scrapping_imdb_top_1000_dataset.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "FhZMybpo6vu7"
      },
      "source": [
        "Project Outline \n",
        "1.   We are going to scrape imdb top 1000 from multiple pages\n",
        "2.   We will get a list of imdb top 1000 sorted by imdb rating descending as of 05-08-2022\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Importing Libraries"
      ],
      "metadata": {
        "id": "LIO4qVwvT9Io"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "Ds0Y0sDh35wX"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from time import sleep\n",
        "from random import randint"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Creating List for column names"
      ],
      "metadata": {
        "id": "D-Lu64WfUDcP"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "movie_name = []\n",
        "year = []\n",
        "time = []\n",
        "rating = []\n",
        "metascore = []\n",
        "votes = []\n",
        "gross = []\n",
        "certificate = []\n",
        "genre = []\n",
        "description = []\n",
        "director = []\n",
        "poster = []"
      ],
      "metadata": {
        "id": "ucct-cNPv4tK"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Use beautiful soup to parse and extract information"
      ],
      "metadata": {
        "id": "6deU01GYUaL4"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "pages = np.arange(1,1000,100)"
      ],
      "metadata": {
        "id": "2ge1HyzuJNwj"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "for page in pages:\n",
        "  page = requests.get('https://www.imdb.com/search/title/?groups=top_1000&sort=user_rating,desc&count=100&start='+str(page)+'&ref_=adv_nxt')\n",
        "  doc = BeautifulSoup(page.text,'html.parser')\n",
        "  movie_data = doc.find_all('div',attrs={'class':'lister-item mode-advanced'})\n",
        "  sleep(randint(2,8))\n",
        "  \n",
        "  for store in movie_data:\n",
        "    name = store.h3.a.text\n",
        "    movie_name.append(name)\n",
        "\n",
        "    release_year = store.h3.find('span', class_='lister-item-year text-muted unbold').text.replace('(','').replace(')','')\n",
        "    year.append(release_year)\n",
        "\n",
        "    run_time = store.p.find('span', class_='runtime').text.replace('\\tmin','')\n",
        "    time.append(run_time)\n",
        "\n",
        "    rate = store.find('div', class_='inline-block ratings-imdb-rating').text.replace('\\n', '')\n",
        "    rating.append(rate)\n",
        "\n",
        "    if store.find('span', class_='metascore') == None: \n",
        "      meta = 'NaN'\n",
        "    else:\n",
        "      meta = store.find('span', class_='metascore').text.strip()\n",
        "    metascore.append(meta)\n",
        "\n",
        "    voting = store.find_all('span',{'name':'nv'})\n",
        "    votes.append(voting[0].text)\n",
        "    grosses = (voting[1].text) if len(voting) >1 else 'NaN'\n",
        "    gross.append(grosses)\n",
        "\n",
        "    if store.find('span',class_='certificate') == None: \n",
        "      certi = 'NaN'\n",
        "    else:\n",
        "      certi = store.find('span',class_='certificate').text.strip()\n",
        "    certificate.append(certi)\n",
        "\n",
        "    type_of_movie = store.find('span', class_='genre').text.strip()\n",
        "    genre.append(type_of_movie)  \n",
        "\n",
        "    desc = store.find_all('p',class_='text-muted')\n",
        "    description.append(desc[1].text.strip())\n",
        "\n",
        "    dir = store.find_all('a')\n",
        "    director.append(dir[-6].text.strip())\n",
        "\n",
        "    posterLink = store.find('img',class_='loadlate')\n",
        "    poster.append(posterLink.attrs['loadlate'])"
      ],
      "metadata": {
        "id": "94HdVDvdJjtw"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "# Create CSV file(s) with the extracted information"
      ],
      "metadata": {
        "id": "1ZD-WoRlUhuT"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "imdb_dataset = pd.DataFrame({'Poster Link':poster, 'Title':movie_name, 'Release Year':year, 'Certificate':certificate,\n",
        "                             'Runtime':time, 'Genre':genre,'Votes':votes, 'Rating':rating, 'Description':description, 'metascore':metascore,\n",
        "                             'Director':director,'Gross':gross})"
      ],
      "metadata": {
        "id": "r0ZSdYAXqrKD"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "imdb_dataset"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "BTHlMcpjS1-j",
        "outputId": "8b3791d2-a4f1-4b07-a1ed-b5975bc271d9"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                           Poster Link  \\\n",
              "0    https://m.media-amazon.com/images/M/MV5BMDFkYT...   \n",
              "1    https://m.media-amazon.com/images/M/MV5BM2MyNj...   \n",
              "2    https://m.media-amazon.com/images/M/MV5BMTMxNT...   \n",
              "3    https://m.media-amazon.com/images/M/MV5BNzA5ZD...   \n",
              "4    https://m.media-amazon.com/images/M/MV5BNDE4OT...   \n",
              "..                                                 ...   \n",
              "995  https://m.media-amazon.com/images/M/MV5BYmFlNT...   \n",
              "996  https://m.media-amazon.com/images/M/MV5BM2U3Yz...   \n",
              "997  https://m.media-amazon.com/images/M/MV5BMTQwMz...   \n",
              "998  https://m.media-amazon.com/images/M/MV5BMTY5OD...   \n",
              "999  https://m.media-amazon.com/images/M/MV5BMTIzZD...   \n",
              "\n",
              "                                             Title Release Year Certificate  \\\n",
              "0                         The Shawshank Redemption         1994           R   \n",
              "1                                    The Godfather         1972           R   \n",
              "2                                  The Dark Knight         2008       PG-13   \n",
              "3    The Lord of the Rings: The Return of the King         2003       PG-13   \n",
              "4                                 Schindler's List         1993           R   \n",
              "..                                             ...          ...         ...   \n",
              "995                                        Sabrina         1954      Passed   \n",
              "996                          From Here to Eternity         1953      Passed   \n",
              "997                Snow White and the Seven Dwarfs         1937    Approved   \n",
              "998                                   The 39 Steps         1935    Approved   \n",
              "999                              The Invisible Man         1933       TV-PG   \n",
              "\n",
              "     Runtime                         Genre      Votes Rating  \\\n",
              "0    142 min                         Drama  2,620,540    9.3   \n",
              "1    175 min                  Crime, Drama  1,814,839    9.2   \n",
              "2    152 min          Action, Crime, Drama  2,592,377    9.0   \n",
              "3    201 min      Action, Adventure, Drama  1,797,943    9.0   \n",
              "4    195 min     Biography, Drama, History  1,330,949    9.0   \n",
              "..       ...                           ...        ...    ...   \n",
              "995  113 min        Comedy, Drama, Romance     64,787    7.6   \n",
              "996  118 min           Drama, Romance, War     46,961    7.6   \n",
              "997   83 min  Animation, Adventure, Family    197,319    7.6   \n",
              "998   86 min      Crime, Mystery, Thriller     56,814    7.6   \n",
              "999   71 min                Horror, Sci-Fi     34,800    7.6   \n",
              "\n",
              "                                           Description metascore  \\\n",
              "0    Two imprisoned men bond over a number of years...        81   \n",
              "1    The aging patriarch of an organized crime dyna...       100   \n",
              "2    When the menace known as the Joker wreaks havo...        84   \n",
              "3    Gandalf and Aragorn lead the World of Men agai...        94   \n",
              "4    In German-occupied Poland during World War II,...        94   \n",
              "..                                                 ...       ...   \n",
              "995  A playboy becomes interested in the daughter o...        72   \n",
              "996  At a U.S. Army base in 1941 Hawaii, a private ...        85   \n",
              "997  Exiled into the dangerous forest by her wicked...        95   \n",
              "998  A man in London tries to help a counter-espion...        93   \n",
              "999  A scientist finds a way of becoming invisible,...        87   \n",
              "\n",
              "                 Director     Gross  \n",
              "0          Frank Darabont   $28.34M  \n",
              "1    Francis Ford Coppola  $134.97M  \n",
              "2       Christopher Nolan  $534.86M  \n",
              "3           Peter Jackson  $377.85M  \n",
              "4        Steven Spielberg   $96.90M  \n",
              "..                    ...       ...  \n",
              "995                     X       NaN  \n",
              "996                     X   $30.50M  \n",
              "997          Perce Pearce  $184.93M  \n",
              "998                     X       NaN  \n",
              "999                     X       NaN  \n",
              "\n",
              "[1000 rows x 12 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e28a9ebc-25b2-43f8-852c-60e766dc7c19\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Poster Link</th>\n",
              "      <th>Title</th>\n",
              "      <th>Release Year</th>\n",
              "      <th>Certificate</th>\n",
              "      <th>Runtime</th>\n",
              "      <th>Genre</th>\n",
              "      <th>Votes</th>\n",
              "      <th>Rating</th>\n",
              "      <th>Description</th>\n",
              "      <th>metascore</th>\n",
              "      <th>Director</th>\n",
              "      <th>Gross</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BMDFkYT...</td>\n",
              "      <td>The Shawshank Redemption</td>\n",
              "      <td>1994</td>\n",
              "      <td>R</td>\n",
              "      <td>142 min</td>\n",
              "      <td>Drama</td>\n",
              "      <td>2,620,540</td>\n",
              "      <td>9.3</td>\n",
              "      <td>Two imprisoned men bond over a number of years...</td>\n",
              "      <td>81</td>\n",
              "      <td>Frank Darabont</td>\n",
              "      <td>$28.34M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BM2MyNj...</td>\n",
              "      <td>The Godfather</td>\n",
              "      <td>1972</td>\n",
              "      <td>R</td>\n",
              "      <td>175 min</td>\n",
              "      <td>Crime, Drama</td>\n",
              "      <td>1,814,839</td>\n",
              "      <td>9.2</td>\n",
              "      <td>The aging patriarch of an organized crime dyna...</td>\n",
              "      <td>100</td>\n",
              "      <td>Francis Ford Coppola</td>\n",
              "      <td>$134.97M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BMTMxNT...</td>\n",
              "      <td>The Dark Knight</td>\n",
              "      <td>2008</td>\n",
              "      <td>PG-13</td>\n",
              "      <td>152 min</td>\n",
              "      <td>Action, Crime, Drama</td>\n",
              "      <td>2,592,377</td>\n",
              "      <td>9.0</td>\n",
              "      <td>When the menace known as the Joker wreaks havo...</td>\n",
              "      <td>84</td>\n",
              "      <td>Christopher Nolan</td>\n",
              "      <td>$534.86M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BNzA5ZD...</td>\n",
              "      <td>The Lord of the Rings: The Return of the King</td>\n",
              "      <td>2003</td>\n",
              "      <td>PG-13</td>\n",
              "      <td>201 min</td>\n",
              "      <td>Action, Adventure, Drama</td>\n",
              "      <td>1,797,943</td>\n",
              "      <td>9.0</td>\n",
              "      <td>Gandalf and Aragorn lead the World of Men agai...</td>\n",
              "      <td>94</td>\n",
              "      <td>Peter Jackson</td>\n",
              "      <td>$377.85M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BNDE4OT...</td>\n",
              "      <td>Schindler's List</td>\n",
              "      <td>1993</td>\n",
              "      <td>R</td>\n",
              "      <td>195 min</td>\n",
              "      <td>Biography, Drama, History</td>\n",
              "      <td>1,330,949</td>\n",
              "      <td>9.0</td>\n",
              "      <td>In German-occupied Poland during World War II,...</td>\n",
              "      <td>94</td>\n",
              "      <td>Steven Spielberg</td>\n",
              "      <td>$96.90M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>995</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BYmFlNT...</td>\n",
              "      <td>Sabrina</td>\n",
              "      <td>1954</td>\n",
              "      <td>Passed</td>\n",
              "      <td>113 min</td>\n",
              "      <td>Comedy, Drama, Romance</td>\n",
              "      <td>64,787</td>\n",
              "      <td>7.6</td>\n",
              "      <td>A playboy becomes interested in the daughter o...</td>\n",
              "      <td>72</td>\n",
              "      <td>X</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>996</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BM2U3Yz...</td>\n",
              "      <td>From Here to Eternity</td>\n",
              "      <td>1953</td>\n",
              "      <td>Passed</td>\n",
              "      <td>118 min</td>\n",
              "      <td>Drama, Romance, War</td>\n",
              "      <td>46,961</td>\n",
              "      <td>7.6</td>\n",
              "      <td>At a U.S. Army base in 1941 Hawaii, a private ...</td>\n",
              "      <td>85</td>\n",
              "      <td>X</td>\n",
              "      <td>$30.50M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>997</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BMTQwMz...</td>\n",
              "      <td>Snow White and the Seven Dwarfs</td>\n",
              "      <td>1937</td>\n",
              "      <td>Approved</td>\n",
              "      <td>83 min</td>\n",
              "      <td>Animation, Adventure, Family</td>\n",
              "      <td>197,319</td>\n",
              "      <td>7.6</td>\n",
              "      <td>Exiled into the dangerous forest by her wicked...</td>\n",
              "      <td>95</td>\n",
              "      <td>Perce Pearce</td>\n",
              "      <td>$184.93M</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>998</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BMTY5OD...</td>\n",
              "      <td>The 39 Steps</td>\n",
              "      <td>1935</td>\n",
              "      <td>Approved</td>\n",
              "      <td>86 min</td>\n",
              "      <td>Crime, Mystery, Thriller</td>\n",
              "      <td>56,814</td>\n",
              "      <td>7.6</td>\n",
              "      <td>A man in London tries to help a counter-espion...</td>\n",
              "      <td>93</td>\n",
              "      <td>X</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>999</th>\n",
              "      <td>https://m.media-amazon.com/images/M/MV5BMTIzZD...</td>\n",
              "      <td>The Invisible Man</td>\n",
              "      <td>1933</td>\n",
              "      <td>TV-PG</td>\n",
              "      <td>71 min</td>\n",
              "      <td>Horror, Sci-Fi</td>\n",
              "      <td>34,800</td>\n",
              "      <td>7.6</td>\n",
              "      <td>A scientist finds a way of becoming invisible,...</td>\n",
              "      <td>87</td>\n",
              "      <td>X</td>\n",
              "      <td>NaN</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>1000 rows × 12 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e28a9ebc-25b2-43f8-852c-60e766dc7c19')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e28a9ebc-25b2-43f8-852c-60e766dc7c19 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e28a9ebc-25b2-43f8-852c-60e766dc7c19');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "imdb_dataset.to_csv('imdb_dataset', index=None)"
      ],
      "metadata": {
        "id": "6zpGEa9BR41n"
      },
      "execution_count": 7,
      "outputs": []
    }
  ],
  "metadata": {
    "colab": {
      "name": "imdb.ipynb",
      "provenance": [],
      "collapsed_sections": [
        "LIO4qVwvT9Io",
        "D-Lu64WfUDcP",
        "6deU01GYUaL4",
        "1ZD-WoRlUhuT"
      ]
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}