1. METIS_Krystkowiak_Krystian_Project_2_Regression - scraping.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "35246f2e",
   "metadata": {},
   "source": [
    "# IMDB TV Rating Predictor - Scraping\n",
    "###### METIS Data Science and Machine Learning Bootcamp 2022 by Krystian Krystkowiak\n",
    "###### project/month(2/7) focus: REGRESSION\n",
    "\n",
    "ABSTRACT\n",
    "- Scraped data, used linear regression, lasso, and ridge regularisation to predict the  IMDB TV show ratings."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "2b305a03",
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "import random\n",
    "import time\n",
    "from fake_useragent import UserAgent\n",
    "from bs4 import BeautifulSoup\n",
    "from datetime import datetime\n",
    "from dateutil.parser import parse\n",
    "import pickle"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f96b7f9d",
   "metadata": {},
   "source": [
    "### scraping functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2f88fc6d",
   "metadata": {},
   "outputs": [],
   "source": [
    "def avg_runtime_f(soup): #counting average runtime\n",
    "    try:\n",
    "        avg_runtime = soup.find(text='Runtime').next.text.split()\n",
    "        avg_runtime = int(avg_runtime[0]) if len(avg_runtime) <= 2 else int(avg_runtime[0])*60 + int(avg_runtime[2])\n",
    "        if avg_runtime > 180:\n",
    "            avg_runtime = 42 #if show is longer than 3 hours set it to 42 (statistic lenght of the show)\n",
    "        if avg_runtime == 1: #if it is one hour make it 60 minutes\n",
    "            avg_runtime = 60 \n",
    "    except:\n",
    "        avg_runtime = 0\n",
    "    return avg_runtime\n",
    "\n",
    "def origin_f(soup): #finding origin\n",
    "    try:\n",
    "        return soup.find(text='Country of origin').next.find('a').text\n",
    "    except: #if there is more then one, select first\n",
    "        return soup.find(text='Countries of origin').next.find('a').text\n",
    "    \n",
    "def creators_f(soup): #finding creators\n",
    "    try:\n",
    "        try:\n",
    "            creators = [i.text for i in soup.find(text='Creators').next.find_all('a')]\n",
    "        except: #if there is only one creator\n",
    "            creators = list([soup.find(text='Creator').next.next.next.next.text])\n",
    "    except: \n",
    "        creators = [] \n",
    "    return creators\n",
    "\n",
    "def count_awards(people): #counting awards for person\n",
    "    x=[]\n",
    "    for i in people:\n",
    "        url_c = 'https://www.imdb.com' + i\n",
    "        response_c = requests.get(url_c)\n",
    "        page_c = response_c.text\n",
    "        soup_c = BeautifulSoup(page_c, \"lxml\")\n",
    "        try:\n",
    "            x.append(int(soup_c.find(class_='desc').text.split(\" \")[2]))\n",
    "        except:\n",
    "            x.append(0)\n",
    "    return x    \n",
    "\n",
    "def creators_a_f(soup): #counting awards for creators\n",
    "    try:\n",
    "        try:\n",
    "            cr = [i['href'].split('?')[0]+'awards' for i in soup.find(text='Creators').next.find_all('a')]\n",
    "        except: #if there is only one creator\n",
    "            cr = [soup.find(text='Creator').next.next.find('a')['href'].split('?')[0]+'awards']        \n",
    "        creators_awards = count_awards(cr)        \n",
    "    except: \n",
    "        creators_awards = []\n",
    "    return creators_awards\n",
    "\n",
    "def stars_f(soup): #finding stars\n",
    "    try:\n",
    "        try:\n",
    "            stars = [i.text for i in soup.find(text='Stars').next.find_all('a')]\n",
    "        except: #if there is only one star\n",
    "            stars = list([soup.find(text='Star').next.next.next.next.text]) # did not find such case, but just to be safe\n",
    "    except: \n",
    "        stars = [] \n",
    "    return stars\n",
    "\n",
    "def stars_a_f(soup): #counting awards for stars\n",
    "    try:\n",
    "        try:\n",
    "            st = [i['href'].split('?')[0]+'awards' for i in soup.find(text='Stars').next.find_all('a')]\n",
    "        except: #if there is only one star\n",
    "            st = [soup.find(text='Star').next.next.find('a')['href'].split('?')[0]+'awards']\n",
    "        stars_awards = count_awards(st)        \n",
    "    except: \n",
    "        stars_awards = []\n",
    "    return stars_awards\n",
    "\n",
    "def company_f(soup):\n",
    "    try:\n",
    "        try:\n",
    "            company = [i.text for i in soup.find(text='Production companies').next.find_all('a')]\n",
    "        except: #if there is only one production company\n",
    "            company = [soup.find(text='Production company').next.text]\n",
    "    except: \n",
    "        company = [] \n",
    "    return company\n",
    "\n",
    "def scrape_shows(show_url_list):    \n",
    "    pipeline_list = []\n",
    "    \n",
    "    for show_url in show_url_list:\n",
    "\n",
    "        time.sleep(.5+2*random.random()) #random intervals before every move\n",
    "        \n",
    "        try:\n",
    "            ua = UserAgent()\n",
    "            user_agent = {'User-agent': ua.random} #random agent for safety\n",
    "\n",
    "            response = requests.get(show_url, headers = user_agent) #page of the show\n",
    "            page = response.text\n",
    "            soup = BeautifulSoup(page, \"lxml\")\n",
    "\n",
    "            url_ep = show_url + '/episodes?season=1' #page with episodes\n",
    "            response_ep = requests.get(url_ep)\n",
    "            page_ep = response_ep.text\n",
    "            soup_ep = BeautifulSoup(page_ep, \"lxml\")    \n",
    "\n",
    "            pipeline_list += [{\n",
    "                \"link\": show_url,\n",
    "                \"imdb_rating\": float(soup.find(class_=\"sc-7ab21ed2-1 jGRxWM\").text),\n",
    "                \"title\": soup.find(class_=\"sc-94726ce4-0 cMYixt\").next.next.text,\n",
    "                \"s01_episodes\": int(len(soup_ep.find_all('div', class_='info'))),\n",
    "                \"avg_runtime\": avg_runtime_f(soup),\n",
    "                \"genres\": [i.text for i in soup.find_all('li', class_=\"ipc-inline-list__item ipc-chip__text\")],\n",
    "                \"rel_date\": parse(' '.join(soup.find(text='Release date').next.find('a').next.text.split()[0:3])),\n",
    "                \"certification\": soup.find(class_=\"sc-8c396aa2-2 itZqyK\").next.next.next.next.text,\n",
    "                \"origin\": origin_f(soup),\n",
    "                \"company\": company_f(soup),\n",
    "                \"creators\": creators_f(soup),\n",
    "                \"creators_a\": creators_a_f(soup),\n",
    "                \"stars\": stars_f(soup),\n",
    "                \"stars_a\": stars_a_f(soup)\n",
    "            }]\n",
    "            \n",
    "            pd.DataFrame(pipeline_list).to_pickle(\"./shows.pkl\") #saving progress\n",
    "            \n",
    "        except:\n",
    "            print('Scraping error: '+show_url)\n",
    "    return pipeline_list  "
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "f0e8f3fa",
   "metadata": {},
   "source": [
    "### list of the shows scrapper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "1a69bf74",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1385"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "imdb_search_urls = ['https://www.imdb.com/search/title/?title_type=tv_series&release_date=2019-01-01,2021-12-31&num_votes=1000,&view=simple&sort=release_date,asc&count=250', 'https://www.imdb.com/search/title/?title_type=tv_series&release_date=2019-01-01,2021-12-31&num_votes=1000,&view=simple&sort=release_date,asc&count=250&start=251&ref_=adv_nxt', 'https://www.imdb.com/search/title/?title_type=tv_series&release_date=2019-01-01,2021-12-31&num_votes=1000,&view=simple&sort=release_date,asc&count=250&start=501&ref_=adv_nxt', 'https://www.imdb.com/search/title/?title_type=tv_series&release_date=2019-01-01,2021-12-31&num_votes=1000,&view=simple&sort=release_date,asc&count=250&start=751&ref_=adv_nxt', 'https://www.imdb.com/search/title/?title_type=tv_series&release_date=2019-01-01,2021-12-31&num_votes=1000,&view=simple&sort=release_date,asc&count=250&start=1001&ref_=adv_nxt', 'https://www.imdb.com/search/title/?title_type=tv_series&release_date=2019-01-01,2021-12-31&num_votes=1000,&view=simple&sort=release_date,asc&count=250&start=1251&ref_=adv_nxt']\n",
    "#250 TV Series, Released between 2019-01-01 and 2021-12-31, Rating Count at least 1,000 (Sorted by Release Date Ascending)\n",
    "\n",
    "show_url_list=[]\n",
    "\n",
    "for imdb_search_url in imdb_search_urls:\n",
    "    \n",
    "    time.sleep(.5+2*random.random()) #random intervals\n",
    "    \n",
    "    response = requests.get(imdb_search_url)\n",
    "    response.status_code  #200 = success!\n",
    "    page = response.text\n",
    "    soup = BeautifulSoup(page, \"lxml\")\n",
    "    urls = ['https://www.imdb.com'+i.parent.find('a')['href'][:-1] for i in soup.find_all(class_=\"lister-item-index unbold text-primary\")]  \n",
    "    \n",
    "    show_url_list +=urls\n",
    "\n",
    "len(show_url_list)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ac958bb3",
   "metadata": {},
   "source": [
    "### shows details scraper"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "id": "0eaabc49",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Scraping error: https://www.imdb.com/title/tt9426194\n",
      "Scraping error: https://www.imdb.com/title/tt13176816\n",
      "Scraping error: https://www.imdb.com/title/tt9654080\n",
      "Scraping error: https://www.imdb.com/title/tt9908860\n",
      "Scraping error: https://www.imdb.com/title/tt9817218\n",
      "Scraping error: https://www.imdb.com/title/tt10243628\n",
      "Scraping error: https://www.imdb.com/title/tt8995604\n",
      "Scraping error: https://www.imdb.com/title/tt8661868\n",
      "Scraping error: https://www.imdb.com/title/tt10098248\n",
      "Scraping error: https://www.imdb.com/title/tt8762206\n",
      "Scraping error: https://www.imdb.com/title/tt10327294\n",
      "Scraping error: https://www.imdb.com/title/tt10681780\n",
      "Scraping error: https://www.imdb.com/title/tt11007186\n",
      "Scraping error: https://www.imdb.com/title/tt11041046\n",
      "Scraping error: https://www.imdb.com/title/tt9104072\n",
      "Scraping error: https://www.imdb.com/title/tt10122474\n",
      "Scraping error: https://www.imdb.com/title/tt11285856\n",
      "Scraping error: https://www.imdb.com/title/tt10726424\n",
      "Scraping error: https://www.imdb.com/title/tt12742136\n",
      "Scraping error: https://www.imdb.com/title/tt10011298\n",
      "Scraping error: https://www.imdb.com/title/tt12112118\n",
      "Scraping error: https://www.imdb.com/title/tt11852724\n",
      "Scraping error: https://www.imdb.com/title/tt7759130\n",
      "Scraping error: https://www.imdb.com/title/tt13929916\n",
      "Scraping error: https://www.imdb.com/title/tt12567072\n",
      "Scraping error: https://www.imdb.com/title/tt9610166\n",
      "Scraping error: https://www.imdb.com/title/tt14061838\n",
      "Scraping error: https://www.imdb.com/title/tt16358754\n",
      "Scraping error: https://www.imdb.com/title/tt15146202\n",
      "Scraping error: https://www.imdb.com/title/tt16401516\n"
     ]
    }
   ],
   "source": [
    "shows = scrape_shows(show_url_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49769911",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1355"
      ]
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(shows) #in that case 29 shows missing is not a significant number"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "22f64940",
   "metadata": {},
   "source": [
    "### pickling shows list to save progress and to follow processing in another notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "987e6ae5",
   "metadata": {},
   "outputs": [],
   "source": [
    "shows.to_pickle(\"./shows.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "ba4024a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "shows_final = pd.read_pickle(\"./shows.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "fefdda41",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>imdb_rating</th>\n",
       "      <th>title</th>\n",
       "      <th>s01_episodes</th>\n",
       "      <th>avg_runtime</th>\n",
       "      <th>genres</th>\n",
       "      <th>rel_date</th>\n",
       "      <th>certification</th>\n",
       "      <th>origin</th>\n",
       "      <th>company</th>\n",
       "      <th>creators</th>\n",
       "      <th>creators_a</th>\n",
       "      <th>stars</th>\n",
       "      <th>stars_a</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://www.imdb.com/title/tt7670568</td>\n",
       "      <td>5.8</td>\n",
       "      <td>The Masked Singer</td>\n",
       "      <td>10</td>\n",
       "      <td>60</td>\n",
       "      <td>[Game-Show, Music, Reality-TV]</td>\n",
       "      <td>2019-01-02</td>\n",
       "      <td>TV-PG</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Smart Dog Media, Fox Alternative Entertainmen...</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Jenny McCarthy-Wahlberg, Ken Jeong, Nicole Sc...</td>\n",
       "      <td>[3, 5, 1]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://www.imdb.com/title/tt8001250</td>\n",
       "      <td>6.2</td>\n",
       "      <td>Siempre Bruja</td>\n",
       "      <td>11</td>\n",
       "      <td>40</td>\n",
       "      <td>[Drama, Fantasy]</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>TV-14</td>\n",
       "      <td>Colombia</td>\n",
       "      <td>[Caracol]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[]</td>\n",
       "      <td>[Sofía Araujo Mejía, Angely Gaviria, Sofía Ara...</td>\n",
       "      <td>[0, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.imdb.com/title/tt8115560</td>\n",
       "      <td>6.6</td>\n",
       "      <td>Tidying Up with Marie Kondo</td>\n",
       "      <td>8</td>\n",
       "      <td>40</td>\n",
       "      <td>[Reality-TV]</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td>TV-PG</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Netflix, The Jackal Group]</td>\n",
       "      <td>[Marie Kondo]</td>\n",
       "      <td>[1]</td>\n",
       "      <td>[Marie Kondo, Charlotte Hervieux, Marie Iida]</td>\n",
       "      <td>[1, 0, 0]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.imdb.com/title/tt8324422</td>\n",
       "      <td>8.1</td>\n",
       "      <td>PEN15</td>\n",
       "      <td>10</td>\n",
       "      <td>30</td>\n",
       "      <td>[Comedy]</td>\n",
       "      <td>2019-02-08</td>\n",
       "      <td>TV-MA</td>\n",
       "      <td>United States</td>\n",
       "      <td>[Awesomeness TV, Odenkirk Provissiero Entertai...</td>\n",
       "      <td>[Anna Konkle, Maya Erskine, Sam Zvibleman]</td>\n",
       "      <td>[1, 1, 4]</td>\n",
       "      <td>[Maya Erskine, Anna Konkle, Melora Walters]</td>\n",
       "      <td>[1, 1, 5]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://www.imdb.com/title/tt8888322</td>\n",
       "      <td>6.6</td>\n",
       "      <td>Bloom</td>\n",
       "      <td>6</td>\n",
       "      <td>60</td>\n",
       "      <td>[Drama, Mystery, Sci-Fi]</td>\n",
       "      <td>2019-01-01</td>\n",
       "      <td></td>\n",
       "      <td>Australia</td>\n",
       "      <td>[Playmaker Media]</td>\n",
       "      <td>[Glen Dolman]</td>\n",
       "      <td>[5]</td>\n",
       "      <td>[Bryan Brown, Phoebe Tonkin, Genevieve Morris]</td>\n",
       "      <td>[11, 0, 1]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                   link  imdb_rating  \\\n",
       "0  https://www.imdb.com/title/tt7670568          5.8   \n",
       "1  https://www.imdb.com/title/tt8001250          6.2   \n",
       "2  https://www.imdb.com/title/tt8115560          6.6   \n",
       "3  https://www.imdb.com/title/tt8324422          8.1   \n",
       "4  https://www.imdb.com/title/tt8888322          6.6   \n",
       "\n",
       "                         title  s01_episodes  avg_runtime  \\\n",
       "0            The Masked Singer            10           60   \n",
       "1                Siempre Bruja            11           40   \n",
       "2  Tidying Up with Marie Kondo             8           40   \n",
       "3                        PEN15            10           30   \n",
       "4                        Bloom             6           60   \n",
       "\n",
       "                           genres   rel_date certification         origin  \\\n",
       "0  [Game-Show, Music, Reality-TV] 2019-01-02         TV-PG  United States   \n",
       "1                [Drama, Fantasy] 2019-01-01         TV-14       Colombia   \n",
       "2                    [Reality-TV] 2019-01-01         TV-PG  United States   \n",
       "3                        [Comedy] 2019-02-08         TV-MA  United States   \n",
       "4        [Drama, Mystery, Sci-Fi] 2019-01-01                    Australia   \n",
       "\n",
       "                                             company  \\\n",
       "0  [Smart Dog Media, Fox Alternative Entertainmen...   \n",
       "1                                          [Caracol]   \n",
       "2                        [Netflix, The Jackal Group]   \n",
       "3  [Awesomeness TV, Odenkirk Provissiero Entertai...   \n",
       "4                                  [Playmaker Media]   \n",
       "\n",
       "                                     creators creators_a  \\\n",
       "0                                          []         []   \n",
       "1                                          []         []   \n",
       "2                               [Marie Kondo]        [1]   \n",
       "3  [Anna Konkle, Maya Erskine, Sam Zvibleman]  [1, 1, 4]   \n",
       "4                               [Glen Dolman]        [5]   \n",
       "\n",
       "                                               stars     stars_a  \n",
       "0  [Jenny McCarthy-Wahlberg, Ken Jeong, Nicole Sc...   [3, 5, 1]  \n",
       "1  [Sofía Araujo Mejía, Angely Gaviria, Sofía Ara...   [0, 0, 0]  \n",
       "2      [Marie Kondo, Charlotte Hervieux, Marie Iida]   [1, 0, 0]  \n",
       "3        [Maya Erskine, Anna Konkle, Melora Walters]   [1, 1, 5]  \n",
       "4     [Bryan Brown, Phoebe Tonkin, Genevieve Morris]  [11, 0, 1]  "
      ]
     },
     "execution_count": 100,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "shows_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "id": "cfae60ad",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1355 entries, 0 to 1354\n",
      "Data columns (total 14 columns):\n",
      " #   Column         Non-Null Count  Dtype         \n",
      "---  ------         --------------  -----         \n",
      " 0   link           1355 non-null   object        \n",
      " 1   imdb_rating    1355 non-null   float64       \n",
      " 2   title          1355 non-null   object        \n",
      " 3   s01_episodes   1355 non-null   int64         \n",
      " 4   avg_runtime    1355 non-null   int64         \n",
      " 5   genres         1355 non-null   object        \n",
      " 6   rel_date       1355 non-null   datetime64[ns]\n",
      " 7   certification  1355 non-null   object        \n",
      " 8   origin         1355 non-null   object        \n",
      " 9   company        1355 non-null   object        \n",
      " 10  creators       1355 non-null   object        \n",
      " 11  creators_a     1355 non-null   object        \n",
      " 12  stars          1355 non-null   object        \n",
      " 13  stars_a        1355 non-null   object        \n",
      "dtypes: datetime64[ns](1), float64(1), int64(2), object(10)\n",
      "memory usage: 148.3+ KB\n"
     ]
    }
   ],
   "source": [
    "shows_final.info()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "metis",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.4 | packaged by conda-forge | (main, Mar 24 2022, 17:43:32) [Clang 12.0.1 ]"
  },
  "vscode": {
   "interpreter": {
    "hash": "8c39b89ecc6aca185f0b8f79042b31abb8688590a55076d3a3d65109d6b85286"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}