03_get_book_details.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import csv\n",
    "from bs4 import BeautifulSoup\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "file = \"lexile/raw/book-reviews/3-willows.html\"\n",
    "with open(file, 'r') as f: \n",
    "        soup = BeautifulSoup(f.read(), 'html.parser')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get All Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "title = soup.h1.text\n",
    "description = soup.find(class_='field field-name-field-one-liner field-type-text field-label-hidden').get_text()\n",
    "csm_age = soup.find(class_=\"csm-green-age\").get_text().replace(\"age \", \"\")[:-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Title: 3 Willows\n",
      "\n",
      "Description: Sisterhood author writes sweet new book about friendship.\n",
      "\n",
      "CSM Recommended Age: 12\n"
     ]
    }
   ],
   "source": [
    "print(\"Title: {}\".format(title))\n",
    "print()\n",
    "print(\"Description: {}\".format(description))\n",
    "print()\n",
    "print(\"CSM Recommended Age: {}\".format(csm_age))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Plot Summary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_summary(soup):\n",
    "    remove_charachters = [\"\\xa0\\n\", \"\\xa0a\", \"\\xa0\"]\n",
    "    plot_sum = soup.find(class_=\"field field-name-field-what-is-story field-type-text-long field-label-hidden\").get_text()\n",
    "    for charachter in remove_charachters:\n",
    "        plot_sum = plot_sum.replace(charachter, \" \")\n",
    "    return plot_sum"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Is It Any Good? "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_csm_review(soup):\n",
    "    remove_charachters = [\"\\xa0\\n\", \"\\xa0a\", \"\\xa0\"]\n",
    "    csm_evaluation = soup.find(class_=\"field field-name-field-any-good field-type-text-long field-label-hidden\").get_text()[:-1]\n",
    "    for charachter in remove_charachters:\n",
    "        csm_evaluation = csm_evaluation.replace(charachter, \" \")\n",
    "    return csm_evaluation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Parent's rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "nan"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def parents_rating(soup): \n",
    "    try:\n",
    "        for s in soup.find_all(class_='user-review-statistics adult'):\n",
    "            adult_rating = s.find(\"div\", attrs={'class': 'stat-wrapper age'})\n",
    "        return adult_rating.text.replace(\"age \", \"\")[:-1]\n",
    "    except:\n",
    "        return np.nan\n",
    "parents_rating(soup)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Children's rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'12'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def kids_rating(soup):\n",
    "    try:\n",
    "        for s in soup.find_all(class_='user-review-statistics child'):\n",
    "            childs_rating = s.find(\"div\", attrs={'class': 'stat-wrapper age'})\n",
    "        return childs_rating.text.replace(\"age \", \"\")[:-1]\n",
    "    except:\n",
    "        return np.nan\n",
    "kids_rating(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Parents need to know that this book from the author of The Sisterhood of the Traveling Pants ultimately delivers positive messages about doing what's right, making good choices, taking chances, and accepting oneself. Along the way, the three young teen protagonists deaI with a borderline eating disorder, an alcoholic parent, a fickle cute boy, and the tribulations of fitting in with the popular crowd. There are crushes, dates, and kisses, but no sex. The book name-drops characters from the author's Traveling Pants books and may appeal most to fans of the series.\\n\""
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def pntk(soup):\n",
    "    remove_charachters = [\"\\xa0\\n\", \"\\xa0a\", \"\\xa0\"]\n",
    "    pntk = soup.find(class_=\"field field-name-field-parents-need-to-know field-type-text-long field-label-hidden\").get_text()\n",
    "    for charachter in remove_charachters:\n",
    "        pntk = pntk.replace(charachter, \" \")\n",
    "    return pntk\n",
    "pntk(soup)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'lexile/raw/book-reviews/'\n",
    "files = os.listdir(path)\n",
    "files = [path + file for file in files]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = []\n",
    "descriptions = [] \n",
    "plot_sums = []\n",
    "csm_evaluations = []\n",
    "need_to_know = []\n",
    "p_rating = []\n",
    "k_rating = []\n",
    "csm_ages = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "for file in files:\n",
    "    try: \n",
    "        with open(file, 'r') as f: \n",
    "            soup = BeautifulSoup(f.read(), 'html.parser')\n",
    "            titles.append(soup.h1.text)\n",
    "            descriptions.append(soup.find(class_='field field-name-field-one-liner field-type-text field-label-hidden').get_text())\n",
    "            csm_ages.append(soup.find(class_=\"csm-green-age\").get_text().replace(\"age \", \"\")[:-1])\n",
    "            plot_sums.append(get_summary(soup))\n",
    "            csm_evaluations.append(get_csm_review(soup))\n",
    "            need_to_know.append(pntk(soup))\n",
    "            p_rating.append(parents_rating(soup))\n",
    "            k_rating.append(kids_rating(soup))\n",
    "    except:\n",
    "        continue"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame({\n",
    "    'title': titles,\n",
    "    'description' : descriptions,\n",
    "    'plot': plot_sums,\n",
    "    'csm_review' : csm_evaluations,\n",
    "    'need_to_know' : need_to_know,\n",
    "    'par_rating' : p_rating,\n",
    "    'kids_rating' : k_rating,\n",
    "    'csm_rating' : csm_ages,\n",
    "})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>plot</th>\n",
       "      <th>csm_review</th>\n",
       "      <th>need_to_know</th>\n",
       "      <th>par_rating</th>\n",
       "      <th>kids_rating</th>\n",
       "      <th>csm_rating</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>5811</td>\n",
       "      <td>The Fold</td>\n",
       "      <td>Insightful and frank novel explores beauty in ...</td>\n",
       "      <td>Joyce has a cute crush that doesn't know she e...</td>\n",
       "      <td>An Na has written a beautiful, poignant coming...</td>\n",
       "      <td>Parents need to know that this book deals with...</td>\n",
       "      <td>12</td>\n",
       "      <td>11</td>\n",
       "      <td>9</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5812</td>\n",
       "      <td>Secrets of the Terra-Cotta Soldier</td>\n",
       "      <td>Boy joins a come-to-life clay soldier in excit...</td>\n",
       "      <td>Ming, a 14-year-old boy in 1970s China, is at ...</td>\n",
       "      <td>SECRETS OF THE TERRA-COTTA SOLDIER is an excit...</td>\n",
       "      <td>Parents need to know that Secrets of the Terra...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "      <td>10</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5813</td>\n",
       "      <td>Fallout: Lois Lane, Book 1</td>\n",
       "      <td>Comics reporter updated as teen in fun, slick ...</td>\n",
       "      <td>As the new girl at Metropolis High, Army brat ...</td>\n",
       "      <td>Author Gwenda Bond has Lois' lingo and attitud...</td>\n",
       "      <td>Parents need to know that Fallout reimagines S...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12</td>\n",
       "      <td>13</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5814</td>\n",
       "      <td>The Return of the King</td>\n",
       "      <td>The exciting and bittersweet conclusion of Fro...</td>\n",
       "      <td>The quest to destroy the One Ring of Power bef...</td>\n",
       "      <td>This concluding volume of Tolkien's three-book...</td>\n",
       "      <td>Parents need to know that if kids are ready to...</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5815</td>\n",
       "      <td>The Random House Book of Poetry for Children</td>\n",
       "      <td>Classic will be the most dog-eared on your shelf.</td>\n",
       "      <td>This treasury of 572 poems, some written but m...</td>\n",
       "      <td>Despite its no-nonsense title, THE RANDOM HOUS...</td>\n",
       "      <td>Parents need to know that this collection of p...</td>\n",
       "      <td>8</td>\n",
       "      <td>NaN</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             title  \\\n",
       "5811                                      The Fold   \n",
       "5812            Secrets of the Terra-Cotta Soldier   \n",
       "5813                    Fallout: Lois Lane, Book 1   \n",
       "5814                        The Return of the King   \n",
       "5815  The Random House Book of Poetry for Children   \n",
       "\n",
       "                                            description  \\\n",
       "5811  Insightful and frank novel explores beauty in ...   \n",
       "5812  Boy joins a come-to-life clay soldier in excit...   \n",
       "5813  Comics reporter updated as teen in fun, slick ...   \n",
       "5814  The exciting and bittersweet conclusion of Fro...   \n",
       "5815  Classic will be the most dog-eared on your shelf.   \n",
       "\n",
       "                                                   plot  \\\n",
       "5811  Joyce has a cute crush that doesn't know she e...   \n",
       "5812  Ming, a 14-year-old boy in 1970s China, is at ...   \n",
       "5813  As the new girl at Metropolis High, Army brat ...   \n",
       "5814  The quest to destroy the One Ring of Power bef...   \n",
       "5815  This treasury of 572 poems, some written but m...   \n",
       "\n",
       "                                             csm_review  \\\n",
       "5811  An Na has written a beautiful, poignant coming...   \n",
       "5812  SECRETS OF THE TERRA-COTTA SOLDIER is an excit...   \n",
       "5813  Author Gwenda Bond has Lois' lingo and attitud...   \n",
       "5814  This concluding volume of Tolkien's three-book...   \n",
       "5815  Despite its no-nonsense title, THE RANDOM HOUS...   \n",
       "\n",
       "                                           need_to_know par_rating  \\\n",
       "5811  Parents need to know that this book deals with...         12   \n",
       "5812  Parents need to know that Secrets of the Terra...        NaN   \n",
       "5813  Parents need to know that Fallout reimagines S...        NaN   \n",
       "5814  Parents need to know that if kids are ready to...         11   \n",
       "5815  Parents need to know that this collection of p...          8   \n",
       "\n",
       "     kids_rating csm_rating  \n",
       "5811          11          9  \n",
       "5812           4         10  \n",
       "5813          12         13  \n",
       "5814          11         12  \n",
       "5815         NaN          5  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5816, 8)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "df.to_csv('csv/book_info.csv')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get All Data from the Summary box"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "\n",
    "for file in files:\n",
    "    with open(file, 'r') as f: \n",
    "        soup = BeautifulSoup(f.read(), 'html.parser') \n",
    "        summary = soup.find(class_=\"shutter-summary-pane panel-pane pane-product-details\")\n",
    "        links = summary.find_all('li')\n",
    "        d = {}\n",
    "        for link in links:\n",
    "            k, v = link.text.split(\":\")\n",
    "            k = k.strip()\n",
    "            v = v.strip()\n",
    "            d[k] = v\n",
    "        data.append(d)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2 = pd.DataFrame(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Author</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Topics</th>\n",
       "      <th>Book type</th>\n",
       "      <th>Publisher</th>\n",
       "      <th>Publication date</th>\n",
       "      <th>Publisher's recommended age(s)</th>\n",
       "      <th>Number of pages</th>\n",
       "      <th>Available on</th>\n",
       "      <th>Last updated</th>\n",
       "      <th>Illustrator</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Awards</th>\n",
       "      <th>Publishers</th>\n",
       "      <th>Award</th>\n",
       "      <th>Illustrators</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>CJ Omololu</td>\n",
       "      <td>Mystery</td>\n",
       "      <td>Adventures, Brothers and Sisters, Friendship, ...</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Delacorte Press</td>\n",
       "      <td>February 24, 2015</td>\n",
       "      <td>12 - 18</td>\n",
       "      <td>336</td>\n",
       "      <td>Nook, Hardback, iBooks, Kindle</td>\n",
       "      <td>June 19, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Beth Kephart</td>\n",
       "      <td>Coming of Age</td>\n",
       "      <td>Friendship, History, Horses and Farm Animals</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Philomel</td>\n",
       "      <td>July 19, 2012</td>\n",
       "      <td>14 - 17</td>\n",
       "      <td>304</td>\n",
       "      <td>Nook, Hardback, iBooks, Kindle</td>\n",
       "      <td>May 06, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>Soman Chainani</td>\n",
       "      <td>Fairy Tale</td>\n",
       "      <td>Magic and Fantasy, Princesses, Fairies, Mermai...</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>HarperCollins Children's Books</td>\n",
       "      <td>May 14, 2013</td>\n",
       "      <td>8 - 17</td>\n",
       "      <td>496</td>\n",
       "      <td>Nook, Audiobook (unabridged), Hardback, iBooks...</td>\n",
       "      <td>October 18, 2017</td>\n",
       "      <td>Iacopo Bruno</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Kami Garcia</td>\n",
       "      <td>Science Fiction</td>\n",
       "      <td>Magic and Fantasy, Adventures, Great Boy Role ...</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Imprint</td>\n",
       "      <td>January 3, 2017</td>\n",
       "      <td>14 - 18</td>\n",
       "      <td>320</td>\n",
       "      <td>Nook, Audiobook (abridged), Hardback, iBooks, ...</td>\n",
       "      <td>June 19, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Tonya Bolden</td>\n",
       "      <td>Historical Fiction</td>\n",
       "      <td>Friendship, History</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Bloomsbury Children's Books</td>\n",
       "      <td>May 30, 2017</td>\n",
       "      <td>NaN</td>\n",
       "      <td>240</td>\n",
       "      <td>Nook, Audiobook (unabridged), Hardback, Kindle</td>\n",
       "      <td>January 18, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           Author               Genre  \\\n",
       "0      CJ Omololu             Mystery   \n",
       "1    Beth Kephart       Coming of Age   \n",
       "2  Soman Chainani          Fairy Tale   \n",
       "3     Kami Garcia     Science Fiction   \n",
       "4    Tonya Bolden  Historical Fiction   \n",
       "\n",
       "                                              Topics Book type  \\\n",
       "0  Adventures, Brothers and Sisters, Friendship, ...   Fiction   \n",
       "1       Friendship, History, Horses and Farm Animals   Fiction   \n",
       "2  Magic and Fantasy, Princesses, Fairies, Mermai...   Fiction   \n",
       "3  Magic and Fantasy, Adventures, Great Boy Role ...   Fiction   \n",
       "4                                Friendship, History   Fiction   \n",
       "\n",
       "                        Publisher   Publication date  \\\n",
       "0                 Delacorte Press  February 24, 2015   \n",
       "1                        Philomel      July 19, 2012   \n",
       "2  HarperCollins Children's Books       May 14, 2013   \n",
       "3                         Imprint    January 3, 2017   \n",
       "4     Bloomsbury Children's Books       May 30, 2017   \n",
       "\n",
       "  Publisher's recommended age(s) Number of pages  \\\n",
       "0                        12 - 18             336   \n",
       "1                        14 - 17             304   \n",
       "2                         8 - 17             496   \n",
       "3                        14 - 18             320   \n",
       "4                            NaN             240   \n",
       "\n",
       "                                        Available on      Last updated  \\\n",
       "0                     Nook, Hardback, iBooks, Kindle     June 19, 2019   \n",
       "1                     Nook, Hardback, iBooks, Kindle      May 06, 2019   \n",
       "2  Nook, Audiobook (unabridged), Hardback, iBooks...  October 18, 2017   \n",
       "3  Nook, Audiobook (abridged), Hardback, iBooks, ...     June 19, 2019   \n",
       "4     Nook, Audiobook (unabridged), Hardback, Kindle  January 18, 2019   \n",
       "\n",
       "    Illustrator Authors Awards Publishers Award Illustrators  \n",
       "0           NaN     NaN    NaN        NaN   NaN          NaN  \n",
       "1           NaN     NaN    NaN        NaN   NaN          NaN  \n",
       "2  Iacopo Bruno     NaN    NaN        NaN   NaN          NaN  \n",
       "3           NaN     NaN    NaN        NaN   NaN          NaN  \n",
       "4           NaN     NaN    NaN        NaN   NaN          NaN  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5816, 16)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "df2.to_csv('csv/book_info_summary.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Author</th>\n",
       "      <th>Genre</th>\n",
       "      <th>Topics</th>\n",
       "      <th>Book type</th>\n",
       "      <th>Publisher</th>\n",
       "      <th>Publication date</th>\n",
       "      <th>Publisher's recommended age(s)</th>\n",
       "      <th>Number of pages</th>\n",
       "      <th>Available on</th>\n",
       "      <th>Last updated</th>\n",
       "      <th>Illustrator</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Awards</th>\n",
       "      <th>Publishers</th>\n",
       "      <th>Award</th>\n",
       "      <th>Illustrators</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>5811</td>\n",
       "      <td>An Na</td>\n",
       "      <td>Coming of Age</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Putnam Juvenile</td>\n",
       "      <td>April 10, 2008</td>\n",
       "      <td>9 - 12</td>\n",
       "      <td>192</td>\n",
       "      <td>NaN</td>\n",
       "      <td>July 12, 2017</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5812</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Historical Fiction</td>\n",
       "      <td>Adventures, Friendship, History, Horses and Fa...</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Amulet Books</td>\n",
       "      <td>January 14, 2014</td>\n",
       "      <td>10 - 12</td>\n",
       "      <td>240</td>\n",
       "      <td>Nook, Hardback, Kindle</td>\n",
       "      <td>June 19, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Ying Chang Compestine, Vinson Compestine</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5813</td>\n",
       "      <td>Gwenda Bond</td>\n",
       "      <td>Science Fiction</td>\n",
       "      <td>Superheroes, Adventures, Great Girl Role Model...</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Switch Press</td>\n",
       "      <td>May 1, 2015</td>\n",
       "      <td>12 - 18</td>\n",
       "      <td>304</td>\n",
       "      <td>Hardback, iBooks, Kindle</td>\n",
       "      <td>June 19, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5814</td>\n",
       "      <td>J.R.R. Tolkien</td>\n",
       "      <td>Fantasy</td>\n",
       "      <td>Magic and Fantasy, Princesses, Fairies, Mermai...</td>\n",
       "      <td>Fiction</td>\n",
       "      <td>Houghton Mifflin Children's Books</td>\n",
       "      <td>October 20, 1955</td>\n",
       "      <td>12 - 14</td>\n",
       "      <td>520</td>\n",
       "      <td>NaN</td>\n",
       "      <td>February 15, 2020</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5815</td>\n",
       "      <td>Jack Prelutsky</td>\n",
       "      <td>Poetry</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Non-Fiction</td>\n",
       "      <td>Random House</td>\n",
       "      <td>September 12, 1983</td>\n",
       "      <td>5 - 7</td>\n",
       "      <td>248</td>\n",
       "      <td>NaN</td>\n",
       "      <td>July 12, 2017</td>\n",
       "      <td>Arnold Lobel</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              Author               Genre  \\\n",
       "5811           An Na       Coming of Age   \n",
       "5812             NaN  Historical Fiction   \n",
       "5813     Gwenda Bond     Science Fiction   \n",
       "5814  J.R.R. Tolkien             Fantasy   \n",
       "5815  Jack Prelutsky              Poetry   \n",
       "\n",
       "                                                 Topics    Book type  \\\n",
       "5811                                                NaN      Fiction   \n",
       "5812  Adventures, Friendship, History, Horses and Fa...      Fiction   \n",
       "5813  Superheroes, Adventures, Great Girl Role Model...      Fiction   \n",
       "5814  Magic and Fantasy, Princesses, Fairies, Mermai...      Fiction   \n",
       "5815                                                NaN  Non-Fiction   \n",
       "\n",
       "                              Publisher    Publication date  \\\n",
       "5811                    Putnam Juvenile      April 10, 2008   \n",
       "5812                       Amulet Books    January 14, 2014   \n",
       "5813                       Switch Press         May 1, 2015   \n",
       "5814  Houghton Mifflin Children's Books    October 20, 1955   \n",
       "5815                       Random House  September 12, 1983   \n",
       "\n",
       "     Publisher's recommended age(s) Number of pages              Available on  \\\n",
       "5811                         9 - 12             192                       NaN   \n",
       "5812                        10 - 12             240    Nook, Hardback, Kindle   \n",
       "5813                        12 - 18             304  Hardback, iBooks, Kindle   \n",
       "5814                        12 - 14             520                       NaN   \n",
       "5815                          5 - 7             248                       NaN   \n",
       "\n",
       "           Last updated   Illustrator  \\\n",
       "5811      July 12, 2017           NaN   \n",
       "5812      June 19, 2019           NaN   \n",
       "5813      June 19, 2019           NaN   \n",
       "5814  February 15, 2020           NaN   \n",
       "5815      July 12, 2017  Arnold Lobel   \n",
       "\n",
       "                                       Authors Awards Publishers Award  \\\n",
       "5811                                       NaN    NaN        NaN   NaN   \n",
       "5812  Ying Chang Compestine, Vinson Compestine    NaN        NaN   NaN   \n",
       "5813                                       NaN    NaN        NaN   NaN   \n",
       "5814                                       NaN    NaN        NaN   NaN   \n",
       "5815                                       NaN    NaN        NaN   NaN   \n",
       "\n",
       "     Illustrators  \n",
       "5811          NaN  \n",
       "5812          NaN  \n",
       "5813          NaN  \n",
       "5814          NaN  \n",
       "5815          NaN  "
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3 = pd.concat([df, df2], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5816, 8)"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5816, 16)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(5816, 24)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>description</th>\n",
       "      <th>plot</th>\n",
       "      <th>csm_review</th>\n",
       "      <th>need_to_know</th>\n",
       "      <th>par_rating</th>\n",
       "      <th>kids_rating</th>\n",
       "      <th>csm_rating</th>\n",
       "      <th>Author</th>\n",
       "      <th>Genre</th>\n",
       "      <th>...</th>\n",
       "      <th>Publisher's recommended age(s)</th>\n",
       "      <th>Number of pages</th>\n",
       "      <th>Available on</th>\n",
       "      <th>Last updated</th>\n",
       "      <th>Illustrator</th>\n",
       "      <th>Authors</th>\n",
       "      <th>Awards</th>\n",
       "      <th>Publishers</th>\n",
       "      <th>Award</th>\n",
       "      <th>Illustrators</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>The Third Twin</td>\n",
       "      <td>Gripping thriller skimps on character developm...</td>\n",
       "      <td>Twins Ava and Alexa \"Lexi\" Rios live in an aff...</td>\n",
       "      <td>THE THIRD TWIN has an interesting, compelling ...</td>\n",
       "      <td>Parents need to know that The Third Twin is a ...</td>\n",
       "      <td>17</td>\n",
       "      <td>14</td>\n",
       "      <td>12</td>\n",
       "      <td>CJ Omololu</td>\n",
       "      <td>Mystery</td>\n",
       "      <td>...</td>\n",
       "      <td>12 - 18</td>\n",
       "      <td>336</td>\n",
       "      <td>Nook, Hardback, iBooks, Kindle</td>\n",
       "      <td>June 19, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Small Damages</td>\n",
       "      <td>Luminous story of pregnant teen's summer in Sp...</td>\n",
       "      <td>It's the summer of 1996, which 18-year-old Ken...</td>\n",
       "      <td>This could well have been a minefield of clich...</td>\n",
       "      <td>Parents need to know that Small Damages is nar...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>14</td>\n",
       "      <td>14</td>\n",
       "      <td>Beth Kephart</td>\n",
       "      <td>Coming of Age</td>\n",
       "      <td>...</td>\n",
       "      <td>14 - 17</td>\n",
       "      <td>304</td>\n",
       "      <td>Nook, Hardback, iBooks, Kindle</td>\n",
       "      <td>May 06, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>The School for Good and Evil, Book 1</td>\n",
       "      <td>Fractured fairy tale has plenty of twists for ...</td>\n",
       "      <td>When best friends Sophie and Agatha are stolen...</td>\n",
       "      <td>The School for Good and Evil is no run-of-the-...</td>\n",
       "      <td>Parents need to know that The School for Good ...</td>\n",
       "      <td>11</td>\n",
       "      <td>11</td>\n",
       "      <td>8</td>\n",
       "      <td>Soman Chainani</td>\n",
       "      <td>Fairy Tale</td>\n",
       "      <td>...</td>\n",
       "      <td>8 - 17</td>\n",
       "      <td>496</td>\n",
       "      <td>Nook, Audiobook (unabridged), Hardback, iBooks...</td>\n",
       "      <td>October 18, 2017</td>\n",
       "      <td>Iacopo Bruno</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Agent of Chaos: The X-Files Origins, Book 1</td>\n",
       "      <td>Series pictures Mulder as teen, captures essen...</td>\n",
       "      <td>Set in 1979, AGENT OF CHAOS follows a 17-year-...</td>\n",
       "      <td>Popular TV characters don't always make a smoo...</td>\n",
       "      <td>Parents need to know that Agent of Chaos: The ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13</td>\n",
       "      <td>Kami Garcia</td>\n",
       "      <td>Science Fiction</td>\n",
       "      <td>...</td>\n",
       "      <td>14 - 18</td>\n",
       "      <td>320</td>\n",
       "      <td>Nook, Audiobook (abridged), Hardback, iBooks, ...</td>\n",
       "      <td>June 19, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>Crossing Ebenezer Creek</td>\n",
       "      <td>Heartbreaking novel follows freed slaves on Sh...</td>\n",
       "      <td>CROSSING EBENEZER CREEK is a YA novel from awa...</td>\n",
       "      <td>Beautifully written and poetically rendered, t...</td>\n",
       "      <td>Parents need to know that Crossing Ebenezer Cr...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>13</td>\n",
       "      <td>Tonya Bolden</td>\n",
       "      <td>Historical Fiction</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>240</td>\n",
       "      <td>Nook, Audiobook (unabridged), Hardback, Kindle</td>\n",
       "      <td>January 18, 2019</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                         title  \\\n",
       "0                               The Third Twin   \n",
       "1                                Small Damages   \n",
       "2         The School for Good and Evil, Book 1   \n",
       "3  Agent of Chaos: The X-Files Origins, Book 1   \n",
       "4                      Crossing Ebenezer Creek   \n",
       "\n",
       "                                         description  \\\n",
       "0  Gripping thriller skimps on character developm...   \n",
       "1  Luminous story of pregnant teen's summer in Sp...   \n",
       "2  Fractured fairy tale has plenty of twists for ...   \n",
       "3  Series pictures Mulder as teen, captures essen...   \n",
       "4  Heartbreaking novel follows freed slaves on Sh...   \n",
       "\n",
       "                                                plot  \\\n",
       "0  Twins Ava and Alexa \"Lexi\" Rios live in an aff...   \n",
       "1  It's the summer of 1996, which 18-year-old Ken...   \n",
       "2  When best friends Sophie and Agatha are stolen...   \n",
       "3  Set in 1979, AGENT OF CHAOS follows a 17-year-...   \n",
       "4  CROSSING EBENEZER CREEK is a YA novel from awa...   \n",
       "\n",
       "                                          csm_review  \\\n",
       "0  THE THIRD TWIN has an interesting, compelling ...   \n",
       "1  This could well have been a minefield of clich...   \n",
       "2  The School for Good and Evil is no run-of-the-...   \n",
       "3  Popular TV characters don't always make a smoo...   \n",
       "4  Beautifully written and poetically rendered, t...   \n",
       "\n",
       "                                        need_to_know par_rating kids_rating  \\\n",
       "0  Parents need to know that The Third Twin is a ...         17          14   \n",
       "1  Parents need to know that Small Damages is nar...        NaN          14   \n",
       "2  Parents need to know that The School for Good ...         11          11   \n",
       "3  Parents need to know that Agent of Chaos: The ...        NaN         NaN   \n",
       "4  Parents need to know that Crossing Ebenezer Cr...        NaN         NaN   \n",
       "\n",
       "  csm_rating          Author               Genre  ...  \\\n",
       "0         12      CJ Omololu             Mystery  ...   \n",
       "1         14    Beth Kephart       Coming of Age  ...   \n",
       "2          8  Soman Chainani          Fairy Tale  ...   \n",
       "3         13     Kami Garcia     Science Fiction  ...   \n",
       "4         13    Tonya Bolden  Historical Fiction  ...   \n",
       "\n",
       "  Publisher's recommended age(s) Number of pages  \\\n",
       "0                        12 - 18             336   \n",
       "1                        14 - 17             304   \n",
       "2                         8 - 17             496   \n",
       "3                        14 - 18             320   \n",
       "4                            NaN             240   \n",
       "\n",
       "                                        Available on      Last updated  \\\n",
       "0                     Nook, Hardback, iBooks, Kindle     June 19, 2019   \n",
       "1                     Nook, Hardback, iBooks, Kindle      May 06, 2019   \n",
       "2  Nook, Audiobook (unabridged), Hardback, iBooks...  October 18, 2017   \n",
       "3  Nook, Audiobook (abridged), Hardback, iBooks, ...     June 19, 2019   \n",
       "4     Nook, Audiobook (unabridged), Hardback, Kindle  January 18, 2019   \n",
       "\n",
       "    Illustrator Authors Awards Publishers Award Illustrators  \n",
       "0           NaN     NaN    NaN        NaN   NaN          NaN  \n",
       "1           NaN     NaN    NaN        NaN   NaN          NaN  \n",
       "2  Iacopo Bruno     NaN    NaN        NaN   NaN          NaN  \n",
       "3           NaN     NaN    NaN        NaN   NaN          NaN  \n",
       "4           NaN     NaN    NaN        NaN   NaN          NaN  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df3.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "df3.to_csv('csv/book_info_complete.csv', index=False)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "base"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}