Web Scraping premier league.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "url = 'https://www.skysports.com/premier-league-table/2011'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "page = requests.get(url)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "200\n"
     ]
    }
   ],
   "source": [
    "print(page.status_code)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "print(page)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(page.text)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from bs4 import BeautifulSoup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(page.text,'html.parser')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(soup.prettify)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a class=\"svg-logo site-header__logo\" href=\"https://www.skysports.com\">\n",
      "<img class=\"svg-logo__image\" onerror=\"this.onerror=null;this.src='/core/img/channels/Sky-Sports-Logo.png'\" src=\"/core/img/channels/Sky-Sports-Logo-transparent.svg\"/>\n",
      "</a>\n"
     ]
    }
   ],
   "source": [
    "print(soup.find('a'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "#print(soup.find_all('a'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a class=\"site-header__col site-header__col4 site-header__nav-button-hit-area\" data-role=\"open-left-hand-nav\" href=\"#no-js-nav\" tabindex=\"1\">\n",
       "<svg class=\"site-header__nav-button\" version=\"1.1\" viewbox=\"0 0 2048 2048\" xml:space=\"preserve\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<rect height=\"256\" width=\"2048\" y=\"384\"></rect>\n",
       "<rect height=\"256\" width=\"2048\" y=\"896\"></rect>\n",
       "<rect height=\"256\" width=\"2048\" y=\"1408\"></rect>\n",
       "</svg>\n",
       "</a>"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "soup.find_all('a')[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "league = soup.find('table',class_='standing-table__table')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "league_table = league.find_all('tbody')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Manchester City 89 64 38 28 5 5 93 29\n",
      "Manchester United 89 56 38 28 5 5 89 33\n",
      "Arsenal 70 25 38 21 7 10 74 49\n",
      "Tottenham Hotspur 69 25 38 20 9 9 66 41\n",
      "Newcastle United 65 5 38 19 8 11 56 51\n",
      "Chelsea 64 19 38 18 10 10 65 46\n",
      "Everton 56 10 38 15 11 12 50 40\n",
      "Liverpool 52 7 38 14 10 14 47 40\n",
      "Fulham 52 -3 38 14 10 14 48 51\n",
      "West Bromwich Albion 47 -7 38 13 8 17 45 52\n",
      "Swansea City 47 -7 38 12 11 15 44 51\n",
      "Norwich City 47 -14 38 12 11 15 52 66\n",
      "Sunderland 45 -1 38 11 12 15 45 46\n",
      "Stoke City 45 -17 38 11 12 15 36 53\n",
      "Wigan Athletic 43 -20 38 11 10 17 42 62\n",
      "Aston Villa 38 -16 38 7 17 14 37 53\n",
      "Queens Park Rangers 37 -23 38 10 7 21 43 66\n",
      "Bolton Wanderers 36 -31 38 10 6 22 46 77\n",
      "Blackburn Rovers 31 -30 38 8 7 23 48 78\n",
      "Wolverhampton Wanderers 25 -42 38 5 10 23 40 82\n"
     ]
    }
   ],
   "source": [
    "for league_teams in league_table:\n",
    "    rows = league_teams.find_all('tr')\n",
    "    for row in rows:\n",
    "        team_names = row.find('td',class_ = 'standing-table__cell standing-table__cell--name').text.strip()\n",
    "        team_points = row.find_all('td',class_ = 'standing-table__cell')[9].text.strip()\n",
    "        team_gd = row.find_all('td',class_ = 'standing-table__cell')[8].text.strip()\n",
    "        team_played = row.find_all('td',class_ = 'standing-table__cell')[2].text.strip()\n",
    "        won = row.find_all('td',class_ = 'standing-table__cell')[3].text.strip()\n",
    "        d = row.find_all('td',class_ = 'standing-table__cell')[4].text.strip()\n",
    "        l = row.find_all('td',class_ = 'standing-table__cell')[5].text.strip()\n",
    "        f = row.find_all('td',class_ = 'standing-table__cell')[6].text.strip()\n",
    "        a = row.find_all('td',class_ = 'standing-table__cell')[7].text.strip()\n",
    "        print(team_names, team_points,team_gd,team_played,won,d,l,f,a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "league_2011 = []\n",
    "\n",
    "for league_teams in league_table:\n",
    "    rows = league_teams.find_all('tr')\n",
    "    for row in rows:\n",
    "        team_names = row.find('td',class_ = 'standing-table__cell standing-table__cell--name').text.strip()\n",
    "        team_points = row.find_all('td',class_ = 'standing-table__cell')[9].text.strip()\n",
    "        team_gd = row.find_all('td',class_ = 'standing-table__cell')[8].text.strip()\n",
    "        league_dict = {'name':team_names,'W':won,'D':d,'L':l,'F':f,'gd':team_gd,\n",
    "                      'pts':team_points}\n",
    "        league_2011.append(league_dict)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "df = pd.DataFrame(league_2011)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>W</th>\n",
       "      <th>D</th>\n",
       "      <th>L</th>\n",
       "      <th>F</th>\n",
       "      <th>gd</th>\n",
       "      <th>pts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Manchester City</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>64</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Manchester United</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>56</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Arsenal</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>25</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tottenham Hotspur</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>25</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Newcastle United</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                name  W   D   L   F  gd pts\n",
       "0    Manchester City  5  10  23  40  64  89\n",
       "1  Manchester United  5  10  23  40  56  89\n",
       "2            Arsenal  5  10  23  40  25  70\n",
       "3  Tottenham Hotspur  5  10  23  40  25  69\n",
       "4   Newcastle United  5  10  23  40   5  65"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>W</th>\n",
       "      <th>D</th>\n",
       "      <th>L</th>\n",
       "      <th>F</th>\n",
       "      <th>gd</th>\n",
       "      <th>pts</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Manchester City</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>64</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Manchester United</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>56</td>\n",
       "      <td>89</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Arsenal</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>25</td>\n",
       "      <td>70</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Tottenham Hotspur</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>25</td>\n",
       "      <td>69</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Newcastle United</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>5</td>\n",
       "      <td>65</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Chelsea</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>19</td>\n",
       "      <td>64</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>Everton</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>10</td>\n",
       "      <td>56</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>Liverpool</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>7</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>Fulham</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-3</td>\n",
       "      <td>52</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>West Bromwich Albion</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-7</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>Swansea City</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-7</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>Norwich City</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-14</td>\n",
       "      <td>47</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>Sunderland</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-1</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>Stoke City</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-17</td>\n",
       "      <td>45</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>Wigan Athletic</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-20</td>\n",
       "      <td>43</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>Aston Villa</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-16</td>\n",
       "      <td>38</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>Queens Park Rangers</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-23</td>\n",
       "      <td>37</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>Bolton Wanderers</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-31</td>\n",
       "      <td>36</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>Blackburn Rovers</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-30</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>Wolverhampton Wanderers</td>\n",
       "      <td>5</td>\n",
       "      <td>10</td>\n",
       "      <td>23</td>\n",
       "      <td>40</td>\n",
       "      <td>-42</td>\n",
       "      <td>25</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       name  W   D   L   F   gd pts\n",
       "0           Manchester City  5  10  23  40   64  89\n",
       "1         Manchester United  5  10  23  40   56  89\n",
       "2                   Arsenal  5  10  23  40   25  70\n",
       "3         Tottenham Hotspur  5  10  23  40   25  69\n",
       "4          Newcastle United  5  10  23  40    5  65\n",
       "5                   Chelsea  5  10  23  40   19  64\n",
       "6                   Everton  5  10  23  40   10  56\n",
       "7                 Liverpool  5  10  23  40    7  52\n",
       "8                    Fulham  5  10  23  40   -3  52\n",
       "9      West Bromwich Albion  5  10  23  40   -7  47\n",
       "10             Swansea City  5  10  23  40   -7  47\n",
       "11             Norwich City  5  10  23  40  -14  47\n",
       "12               Sunderland  5  10  23  40   -1  45\n",
       "13               Stoke City  5  10  23  40  -17  45\n",
       "14           Wigan Athletic  5  10  23  40  -20  43\n",
       "15              Aston Villa  5  10  23  40  -16  38\n",
       "16      Queens Park Rangers  5  10  23  40  -23  37\n",
       "17         Bolton Wanderers  5  10  23  40  -31  36\n",
       "18         Blackburn Rovers  5  10  23  40  -30  31\n",
       "19  Wolverhampton Wanderers  5  10  23  40  -42  25"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}