AmbitionBox 20-04-2020.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Web Scraping \"List of companies in India\" from AmbitionBox Website"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Import Packages"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import time\n",
    "from datetime import timedelta"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Request and Fetch the Webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Response [403]>"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# hit \"www.ambitionbox.com/list-of-companies\"\n",
    "requests.get(\"https://www.ambitionbox.com/list-of-companies?page=1\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<HTML><HEAD>\\n<TITLE>Access Denied</TITLE>\\n</HEAD><BODY>\\n<H1>Access Denied</H1>\\n \\nYou don\\'t have permission to access \"http&#58;&#47;&#47;www&#46;ambitionbox&#46;com&#47;list&#45;of&#45;companies&#63;\" on this server.<P>\\nReference&#32;&#35;18&#46;3035d917&#46;1587481164&#46;184b9b9f\\n</BODY>\\n</HTML>\\n'"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# webpage's robot.txt doesn't allow bots!\n",
    "requests.get(\"https://www.ambitionbox.com/list-of-companies?page=1\").text"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "Access Denied\n",
    "You don't have permission to access!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# google chrome browser's request header (to make it look like, we are making this request from a browser)\n",
    "header = {\n",
    "    \"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36\"\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# hit using the header\n",
    "response = requests.get(\"https://www.ambitionbox.com/list-of-companies?page=1\", headers=header)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<!doctype html>\\n<html data-n-head-ssr lang=\"en\" data-n-head=\"%7B%22lang%22:%7B%22ssr%22:%22en%22%7D%7D\">\\n  <head >\\n    <meta charset=\"UTF-8\">\\n    <meta name=\"viewport\" content=\"width=device-width,initial-scale=1\">\\n    <meta http-equiv=\"X-UA-Compatible\" content=\"IE=edge\"> \\n    \\n    <script type=\"text/javascript\">window.NREUM||(NREUM={}),NREUM.init={distributed_tracing:{enabled:!0}},window.NREUM||(NREUM={}),__nr_require=function(n,r,t){function o(e){if(!r[e]){var t=r[e]={exports:{}};n[e][0].call(t'"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# see the recieved page source\n",
    "response.text[0:500]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Pass the fetched webpage response to Beautiful Soup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# give the webpage to Beautiful Soup using parsers: \"html.parser\" or \"lxml\"\n",
    "soup = BeautifulSoup(response.text, 'lxml')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# we see the whole webpage is made of cards and each card has the company info\n",
    "# on inspecting, we see the cards are HTML: \"div\"s with class-name = \"company-content-wrapper\"\n",
    "# Let us extract the first card and see how we can extract data from it...\n",
    "\n",
    "first_company_card = soup.find(\"div\", class_=\"company-content-wrapper\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"company-content-wrapper\">\n",
      " <div class=\"company-content\">\n",
      "  <div class=\"company-logo\">\n",
      "   <img alt=\"Tata Consultancy Services logo\" class=\"lazy\" data-src=\"https://static.ambitionbox.com/alpha/company/photos/logos/tcs.jpg\" onerror=\"this.onerror=null;this.src='/static/icons/company-placeholder.svg';\" src=\"https://static.ambitionbox.com/static/icons/company-placeholder.svg\"/>\n",
      "  </div>\n",
      "  <div class=\"company-info-wrapper\">\n",
      "   <div class=\"company-info\">\n",
      "    <a href=\"/overview/tcs-overview?utm_campaign=lc_card&amp;utm_source=ambitionbox&amp;utm_medium=desktop\">\n",
      "     <h2 class=\"company-name bold-title-l\" title=\"TCS\">\n",
      "      TCS\n",
      "     </h2>\n",
      "    </a>\n",
      "    <div class=\"rating-wrapper\">\n",
      "     <p class=\"rating badge-large rating-35\">\n",
      "      <i class=\"icon icon-star\">\n",
      "      </i>\n",
      "      3.9\n",
      "     </p>\n",
      "     <a class=\"review-count sbold-Labels\" href=\"https://www.ambitionbox.com/reviews/tcs-reviews?utm_campaign=lc_ratings&amp;utm_source=ambitionbox&amp;utm_medium=desktop\">\n",
      "      <!-- -->\n",
      "      16.1k Reviews\n",
      "     </a>\n",
      "    </div>\n",
      "   </div>\n",
      "   <div class=\"company-basic-info\">\n",
      "    <p class=\"infoEntity sbold-list-header\">\n",
      "     <i class=\"icon-domain\">\n",
      "     </i>\n",
      "     Public\n",
      "    </p>\n",
      "    <p class=\"infoEntity sbold-list-header\">\n",
      "     <i class=\"icon-pin-drop\">\n",
      "     </i>\n",
      "     Mumbai + 156 more\n",
      "    </p>\n",
      "    <p class=\"infoEntity sbold-list-header\">\n",
      "     <i class=\"icon-access-time\">\n",
      "     </i>\n",
      "     52 years old\n",
      "    </p>\n",
      "    <p class=\"infoEntity sbold-list-header\">\n",
      "     <i class=\"icon-supervisor-account\">\n",
      "     </i>\n",
      "     10000+ employees\n",
      "    </p>\n",
      "   </div>\n",
      "   <div class=\"chips-block\">\n",
      "    <a class=\"ab_chip body-medium\" href=\"/consulting-companies-in-india\" title=\"Consulting companies in india\">\n",
      "     Consulting\n",
      "    </a>\n",
      "    <a class=\"ab_chip body-medium\" href=\"/it-or-ites-companies-in-india\" title=\"IT/ITES companies in india\">\n",
      "     IT/ITES\n",
      "    </a>\n",
      "    <a class=\"ab_chip body-medium\" href=\"/mnc-companies-in-india\" title=\"MNC companies in india\">\n",
      "     MNC\n",
      "    </a>\n",
      "    <a class=\"ab_chip body-medium\" href=\"/fortune500-companies-in-india\" title=\"Fortune500 companies in india\">\n",
      "     Fortune500\n",
      "    </a>\n",
      "   </div>\n",
      "  </div>\n",
      " </div>\n",
      " <!-- -->\n",
      " <p class=\"description body-small\" itemprop=\"description\">\n",
      "  Tata Consultancy Services is an IT services, consulting and business solutions organisation that has been partnering with the world’s largest businesses in their transformation journeys for the last 50 years. \n",
      "\n",
      "A part of the Tata group, India's largest multinational business group, TCS has over 436,000 of the world’s best-trained consultants in 46 countries. The company is listed on the BSE (formerly Bombay Stock Exchange) and the NSE (National Stock Exchange) in India. \n",
      "\n",
      "TCS'​ proactive stance on climate change and award winning work with communities across the world have earned it a place in leading sustainability indices such as the Dow Jones Sustainability Index (DJSI), MSCI Global Sustainability Index and the FTSE4Good Emerging Index.\n",
      " </p>\n",
      "</div>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# let's see what we got here...\n",
    "print(first_company_card.prettify())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Let us try and extract the data from a single company card"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [1] company name"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<h2 class=\"company-name bold-title-l\" title=\"TCS\">\n",
       "                TCS\n",
       "              </h2>"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 1. company name is inside h2 tag: <h2 class=\"company-name bold-title-l\" title=\"TCS\">\n",
    "first_company_card.find(\"h2\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'TCS'"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract company name\n",
    "first_company_card.find(\"h2\").text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [2] company rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p class=\"rating badge-large rating-35\"><i class=\"icon icon-star\"></i>\n",
       "                 3.9\n",
       "              </p>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2. company rating lies inside p tag: <p class=\"rating badge-large rating-35\">\n",
    "first_company_card.find(\"p\", class_=\"rating\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'3.9'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract company rating\n",
    "first_company_card.find(\"p\", class_=\"rating\").text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [3] number of company reviews"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<a class=\"review-count sbold-Labels\" href=\"https://www.ambitionbox.com/reviews/tcs-reviews?utm_campaign=lc_ratings&amp;utm_source=ambitionbox&amp;utm_medium=desktop\"><!-- -->\n",
       "                16.1k Reviews\n",
       "              </a>"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 3. number of company reviews lies inside a tag: <a class=\"review-count sbold-Labels\">\n",
    "first_company_card.find(\"a\", class_=\"review-count sbold-Labels\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'16.1k Reviews'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract number of company reviews\n",
    "first_company_card.find(\"a\", class_=\"review-count sbold-Labels\").text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'16.1k'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_company_card.find(\"a\", class_=\"review-count sbold-Labels\").text.strip().replace(\" Reviews\", \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [4] domain, [5] location, [6] years old, [7] employee strength"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now this is tricky!\n",
    "# extract \"infoEntity\" containing: \n",
    "# 4. 'domain', \n",
    "# 5. 'location', \n",
    "# 6. 'years old'\n",
    "# 7. 'employee strength'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"infoEntity sbold-list-header\"><i class=\"icon-domain\"></i>\n",
       "               Public\n",
       "             </p>,\n",
       " <p class=\"infoEntity sbold-list-header\"><i class=\"icon-pin-drop\"></i>\n",
       "               Mumbai + 156 more\n",
       "             </p>,\n",
       " <p class=\"infoEntity sbold-list-header\"><i class=\"icon-access-time\"></i>\n",
       "               52 years old\n",
       "             </p>,\n",
       " <p class=\"infoEntity sbold-list-header\"><i class=\"icon-supervisor-account\"></i>\n",
       "               10000+ employees\n",
       "             </p>]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Try 1:\n",
    "first_company_card.find_all(\"p\", class_=\"infoEntity sbold-list-header\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p class=\"infoEntity sbold-list-header\"><i class=\"icon-domain\"></i>\n",
       "              Public\n",
       "            </p>"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "first_company_card.find_all(\"p\", class_=\"infoEntity sbold-list-header\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<p class=\"infoEntity sbold-list-header\"><i class=\"icon-domain\"></i>\n",
       "               Public\n",
       "             </p>,\n",
       " <p class=\"infoEntity sbold-list-header\"><i class=\"icon-pin-drop\"></i>\n",
       "               Mumbai + 156 more\n",
       "             </p>,\n",
       " <p class=\"infoEntity sbold-list-header\"><i class=\"icon-access-time\"></i>\n",
       "               52 years old\n",
       "             </p>,\n",
       " <p class=\"infoEntity sbold-list-header\"><i class=\"icon-supervisor-account\"></i>\n",
       "               10000+ employees\n",
       "             </p>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inner_company_info_list = first_company_card.find_all(\"p\", class_=\"infoEntity sbold-list-header\")\n",
    "inner_company_info_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p class=\"infoEntity sbold-list-header\"><i class=\"icon-domain\"></i>\n",
       "              Public\n",
       "            </p>"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inner_company_info_list[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'icon-supervisor-account'"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inner_company_info_list[3].findChildren(\"i\")[0][\"class\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'10000+ employees'"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inner_company_info_list[3].text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "# works, but not sure if it can be iterated through..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"company-basic-info\">\n",
      " <p class=\"infoEntity sbold-list-header\">\n",
      "  <i class=\"icon-domain\">\n",
      "  </i>\n",
      "  Public\n",
      " </p>\n",
      " <p class=\"infoEntity sbold-list-header\">\n",
      "  <i class=\"icon-pin-drop\">\n",
      "  </i>\n",
      "  Mumbai + 156 more\n",
      " </p>\n",
      " <p class=\"infoEntity sbold-list-header\">\n",
      "  <i class=\"icon-access-time\">\n",
      "  </i>\n",
      "  52 years old\n",
      " </p>\n",
      " <p class=\"infoEntity sbold-list-header\">\n",
      "  <i class=\"icon-supervisor-account\">\n",
      "  </i>\n",
      "  10000+ employees\n",
      " </p>\n",
      "</div>\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Try 2\n",
    "\n",
    "# let's try using it's parent tag: div\n",
    "inner_company_info_card = first_company_card.find(\"div\", class_=\"company-basic-info\")\n",
    "print(inner_company_info_card.prettify())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<i class=\"icon-domain\"></i>,\n",
       " <i class=\"icon-pin-drop\"></i>,\n",
       " <i class=\"icon-access-time\"></i>,\n",
       " <i class=\"icon-supervisor-account\"></i>]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inner_company_info_card.findChildren(\"i\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'icon-domain'"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inner_company_info_card.findChildren(\"i\")[0][\"class\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Public'"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "inner_company_info_card.find_all(\"p\")[0].text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "# This works... let's put it in a single cell code !!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p class=\"infoEntity sbold-list-header\"><i class=\"icon-domain\"></i>\n",
       "              Public\n",
       "            </p>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# ignore\n",
    "first_company_card.find_all(\"p\", class_=\"infoEntity sbold-list-header\")[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "domain: Public\n",
      "location: Mumbai + 156 more\n",
      "years old: 52 years old\n",
      "employee strength: 10000+ employees\n"
     ]
    }
   ],
   "source": [
    "# extract \"infoEntity\" containing 'domain', 'location', 'years old' & 'employee strength'\n",
    "info_list = first_company_card.find_all(\"p\", class_=\"infoEntity sbold-list-header\")\n",
    "dom = None\n",
    "loc = None\n",
    "old = None\n",
    "emp = None\n",
    "\n",
    "for i in range(4):\n",
    "    if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-domain':\n",
    "        dom = info_list[i].text.strip()\n",
    "        \n",
    "    if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-pin-drop':\n",
    "        loc = info_list[i].text.strip()\n",
    "        \n",
    "    if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-access-time':\n",
    "        old = info_list[i].text.strip()\n",
    "        \n",
    "    if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-supervisor-account':\n",
    "        emp = info_list[i].text.strip()\n",
    "    \n",
    "print(\"domain:\", dom)\n",
    "print(\"location:\", loc)\n",
    "print(\"years old:\", old)\n",
    "print(\"employee strength:\", emp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [8] company tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<a class=\"ab_chip body-medium\" href=\"/consulting-companies-in-india\" title=\"Consulting companies in india\">\n",
       "               Consulting\n",
       "             </a>,\n",
       " <a class=\"ab_chip body-medium\" href=\"/it-or-ites-companies-in-india\" title=\"IT/ITES companies in india\">\n",
       "               IT/ITES\n",
       "             </a>,\n",
       " <a class=\"ab_chip body-medium\" href=\"/mnc-companies-in-india\" title=\"MNC companies in india\">\n",
       "               MNC\n",
       "             </a>,\n",
       " <a class=\"ab_chip body-medium\" href=\"/fortune500-companies-in-india\" title=\"Fortune500 companies in india\">\n",
       "               Fortune500\n",
       "             </a>]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 8. company tags are inside a tags: <a class=\"ab_chip\">\n",
    "first_company_card.find_all(\"a\", class_=\"ab_chip\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Consulting', 'IT/ITES', 'MNC', 'Fortune500']"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract company tags\n",
    "tags = []\n",
    "for tag in first_company_card.find_all(\"a\", class_=\"ab_chip\"):\n",
    "    tags.append(tag.text.strip())\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'Consulting, IT/ITES, MNC, Fortune500'"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tags = ', '.join(tags)\n",
    "tags"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### [9] company description"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<p class=\"description body-small\" itemprop=\"description\">Tata Consultancy Services is an IT services, consulting and business solutions organisation that has been partnering with the world’s largest businesses in their transformation journeys for the last 50 years. \n",
       "\n",
       "A part of the Tata group, India's largest multinational business group, TCS has over 436,000 of the world’s best-trained consultants in 46 countries. The company is listed on the BSE (formerly Bombay Stock Exchange) and the NSE (National Stock Exchange) in India. \n",
       "\n",
       "TCS'​ proactive stance on climate change and award winning work with communities across the world have earned it a place in leading sustainability indices such as the Dow Jones Sustainability Index (DJSI), MSCI Global Sustainability Index and the FTSE4Good Emerging Index. \n",
       "</p>"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 9. company description is inside p tag: <p class=\"description\">\n",
    "first_company_card.find(\"p\", class_=\"description\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "\"Tata Consultancy Services is an IT services, consulting and business solutions organisation that has been partnering with the world’s largest businesses in their transformation journeys for the last 50 years. \\n\\nA part of the Tata group, India's largest multinational business group, TCS has over 436,000 of the world’s best-trained consultants in 46 countries. The company is listed on the BSE (formerly Bombay Stock Exchange) and the NSE (National Stock Exchange) in India. \\n\\nTCS'\\u200b proactive stance on climate change and award winning work with communities across the world have earned it a place in leading sustainability indices such as the Dow Jones Sustainability Index (DJSI), MSCI Global Sustainability Index and the FTSE4Good Emerging Index.\""
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# extract company description\n",
    "first_company_card.find(\"p\", class_=\"description\").text.strip()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scraping a single webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Now, Let's put it all together and scrape the webpage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "30"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# find all the company cards in the webpage (HTML divs that encloses data about each company)\n",
    "company_cards = soup.find_all(\"div\", class_=\"company-content-wrapper\")\n",
    "len(company_cards)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Wall time: 20.6 ms\n"
     ]
    }
   ],
   "source": [
    "%%time\n",
    "\n",
    "name = []\n",
    "rating = []\n",
    "reviews = []\n",
    "domain = []\n",
    "location = []\n",
    "years_old = []\n",
    "employee_strength = []\n",
    "tags = []\n",
    "about = []\n",
    "\n",
    "for card in company_cards:\n",
    "    # 1. name\n",
    "    name.append(card.find(\"h2\").text.strip())\n",
    "    \n",
    "    # 2. rating\n",
    "    rating.append(card.find(\"p\", class_=\"rating\").text.strip())\n",
    "    \n",
    "    # 3. reviews\n",
    "    reviews.append(card.find(\"a\", class_=\"review-count sbold-Labels\").text.strip().replace(\" Reviews\", \"\"))\n",
    "    \n",
    "    # 4. domain, 5. location, 6. years old & 7. employee strength\n",
    "    info_list = card.find_all(\"p\", class_=\"infoEntity sbold-list-header\")\n",
    "    dom = None\n",
    "    loc = None\n",
    "    old = None\n",
    "    emp = None\n",
    "    for i in range(4):\n",
    "        try:\n",
    "            if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-domain':\n",
    "                dom = info_list[i].text.strip()\n",
    "\n",
    "            if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-pin-drop':\n",
    "                loc = info_list[i].text.strip()\n",
    "\n",
    "            if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-access-time':\n",
    "                old = info_list[i].text.strip()\n",
    "\n",
    "            if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-supervisor-account':\n",
    "                emp = info_list[i].text.strip()\n",
    "        except:\n",
    "            pass\n",
    "    \n",
    "    domain.append(dom)\n",
    "    location.append(loc)\n",
    "    years_old.append(old)\n",
    "    employee_strength.append(emp)\n",
    "    \n",
    "    # 8. tags\n",
    "    t = []\n",
    "    for tag in card.find_all(\"a\", class_=\"ab_chip\"):\n",
    "        t.append(tag.text.strip())\n",
    "    t = ', '.join(t)\n",
    "    tags.append(t)\n",
    "    \n",
    "    # 9. about\n",
    "    about.append(card.find(\"p\", class_=\"description\").text.strip())\n",
    "    \n",
    "col_dic = {\n",
    "    \"name\": name,\n",
    "    \"rating\": rating,\n",
    "    \"reviews\": reviews,\n",
    "    \"domain\": domain,\n",
    "    \"location\": location,\n",
    "    \"years_old\": years_old,\n",
    "    \"employee_strength\": employee_strength,\n",
    "    \"tags\": tags,\n",
    "    \"about\": about\n",
    "}\n",
    "\n",
    "df = pd.DataFrame(col_dic)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>rating</th>\n",
       "      <th>reviews</th>\n",
       "      <th>domain</th>\n",
       "      <th>location</th>\n",
       "      <th>years_old</th>\n",
       "      <th>employee_strength</th>\n",
       "      <th>tags</th>\n",
       "      <th>about</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>TCS</td>\n",
       "      <td>3.9</td>\n",
       "      <td>16.1k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 156 more</td>\n",
       "      <td>52 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC, Fortune500</td>\n",
       "      <td>Tata Consultancy Services is an IT services, c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Accenture</td>\n",
       "      <td>4.0</td>\n",
       "      <td>14.1k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Dublin + 87 more</td>\n",
       "      <td>31 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC</td>\n",
       "      <td>Accenture is a leading global professional ser...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>ICICI Bank</td>\n",
       "      <td>4.1</td>\n",
       "      <td>12.7k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 724 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Servic...</td>\n",
       "      <td>ICICI Bank is India's largest private sector b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Cognizant</td>\n",
       "      <td>3.9</td>\n",
       "      <td>12.1k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Teaneck + 44 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>IT/ITES, MNC</td>\n",
       "      <td>Cognizant (NASDAQ-100: CTSH) is one of the wor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>HDFC Bank</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10.9k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 692 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Servic...</td>\n",
       "      <td>HDFC Bank is one of India’s leading private ba...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>5</td>\n",
       "      <td>Infosys</td>\n",
       "      <td>3.9</td>\n",
       "      <td>10.8k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Bangalore + 64 more</td>\n",
       "      <td>39 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>IT/ITES, Fortune500, MNC</td>\n",
       "      <td>Infosys is a global leader in next-generation ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>6</td>\n",
       "      <td>L&amp;T</td>\n",
       "      <td>4.1</td>\n",
       "      <td>10.2k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 333 more</td>\n",
       "      <td>82 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Construction, Fortune500, Conglomerate</td>\n",
       "      <td>Larsen &amp; Toubro is a major technology, enginee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>7</td>\n",
       "      <td>Capgemini</td>\n",
       "      <td>3.3</td>\n",
       "      <td>9.6k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Paris + 42 more</td>\n",
       "      <td>53 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Computer Software, Consulting, IT/ITES, MNC</td>\n",
       "      <td>A global leader in consulting, technology serv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>8</td>\n",
       "      <td>Tech Mahindra</td>\n",
       "      <td>3.5</td>\n",
       "      <td>9.1k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Pune + 138 more</td>\n",
       "      <td>34 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC, Fortune500</td>\n",
       "      <td>Tech Mahindra represents the connected world, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>9</td>\n",
       "      <td>HCL Technologies</td>\n",
       "      <td>3.7</td>\n",
       "      <td>8.7k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Noida + 106 more</td>\n",
       "      <td>14 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, Fortune500, MNC</td>\n",
       "      <td>HCL Learning, a wholly owned subsidiary of HCL...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>10</td>\n",
       "      <td>Tata Motors</td>\n",
       "      <td>4.1</td>\n",
       "      <td>8.3k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Pune + 336 more</td>\n",
       "      <td>75 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Automobile, Manufacturing, Fortune500, MNC</td>\n",
       "      <td>Tata Motors Limited, a USD 45 billion organisa...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>11</td>\n",
       "      <td>IBM</td>\n",
       "      <td>4.0</td>\n",
       "      <td>8.2k</td>\n",
       "      <td>Private</td>\n",
       "      <td>New York + 69 more</td>\n",
       "      <td>109 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC</td>\n",
       "      <td>At IBM, work is more than a job - it's a calli...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>12</td>\n",
       "      <td>Reliance jio</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.9k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Navi Mumbai + 588 more</td>\n",
       "      <td>10 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Telecom/ISP, MNC</td>\n",
       "      <td>Make a new start. A start that will build beau...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>13</td>\n",
       "      <td>Genpact</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.7k</td>\n",
       "      <td>Private</td>\n",
       "      <td>New York + 31 more</td>\n",
       "      <td>23 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC</td>\n",
       "      <td>Genpact (NYSE: G) is a global professional ser...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>14</td>\n",
       "      <td>Axis Bank</td>\n",
       "      <td>4.0</td>\n",
       "      <td>7.6k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 645 more</td>\n",
       "      <td>27 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Servic...</td>\n",
       "      <td>Axis Bank is the third largest private sector ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>15</td>\n",
       "      <td>Concentrix Daksh Service...</td>\n",
       "      <td>4.0</td>\n",
       "      <td>6.5k</td>\n",
       "      <td>None</td>\n",
       "      <td>New Delhi + 36 more</td>\n",
       "      <td>21 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>BPO/KPO</td>\n",
       "      <td>A wholly owned subsidiary of SYNNEX Corporatio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>16</td>\n",
       "      <td>Wipro</td>\n",
       "      <td>3.8</td>\n",
       "      <td>6.5k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Bangalore + 116 more</td>\n",
       "      <td>75 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC, Conglomerate, Fortun...</td>\n",
       "      <td>Wipro Limited (NYSE: WIT, BSE: 507685, NSE: WI...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>17</td>\n",
       "      <td>Vodafone</td>\n",
       "      <td>4.3</td>\n",
       "      <td>6k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Berkshire + 366 more</td>\n",
       "      <td>38 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Telecom/ISP, MNC, Fortune500</td>\n",
       "      <td>Vodafone is a leader in technology communicati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>18</td>\n",
       "      <td>HDB Financial Services</td>\n",
       "      <td>4.0</td>\n",
       "      <td>5.5k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Ahmedabad + 434 more</td>\n",
       "      <td>13 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Financial Services</td>\n",
       "      <td>HDB Financial Services (HDBFS) is a leading No...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>19</td>\n",
       "      <td>Bharti Airtel</td>\n",
       "      <td>4.2</td>\n",
       "      <td>5.5k</td>\n",
       "      <td>Public</td>\n",
       "      <td>New Delhi + 339 more</td>\n",
       "      <td>25 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Telecom/ISP, Fortune500</td>\n",
       "      <td>Bharti Airtel is a leading alternative provide...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>20</td>\n",
       "      <td>Kotak Mahindra Bank</td>\n",
       "      <td>3.8</td>\n",
       "      <td>5.2k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 344 more</td>\n",
       "      <td>17 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Servic...</td>\n",
       "      <td>In February 2003, Kotak Mahindra Finance Ltd. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>21</td>\n",
       "      <td>Amazon</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4.9k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Washington + 157 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consumer goods, Internet, Retail, MNC</td>\n",
       "      <td>Amazon is guided by four principles: customer ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>22</td>\n",
       "      <td>Reliance</td>\n",
       "      <td>4.2</td>\n",
       "      <td>4.6k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 206 more</td>\n",
       "      <td>54 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Oil/Gas/Petro Chemicals, Power &amp; Energy, Fortu...</td>\n",
       "      <td>Reliance Industries Limited is a Fortune 500 c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>23</td>\n",
       "      <td>Bajaj Finserv</td>\n",
       "      <td>4.1</td>\n",
       "      <td>3.8k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Pune + 453 more</td>\n",
       "      <td>13 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Financial Services, Fortune500</td>\n",
       "      <td>Bajaj Finserv was formed in April 2007 as a re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>24</td>\n",
       "      <td>IndusInd Bank</td>\n",
       "      <td>3.6</td>\n",
       "      <td>3.7k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 354 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Servic...</td>\n",
       "      <td>IndusInd Bank is a Mumbai based scheduled priv...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>25</td>\n",
       "      <td>Samsung</td>\n",
       "      <td>4.2</td>\n",
       "      <td>3.5k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Seoul + 286 more</td>\n",
       "      <td>82 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consumer goods, Manufacturing, Electronics, Co...</td>\n",
       "      <td>Samsung Electronics is a global leader in tech...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>26</td>\n",
       "      <td>Hinduja Global Solutions</td>\n",
       "      <td>3.8</td>\n",
       "      <td>3.5k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Bangalore + 48 more</td>\n",
       "      <td>20 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>BPO/KPO, MNC, Fortune500</td>\n",
       "      <td>A global leader in business process management...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>27</td>\n",
       "      <td>Indian Army</td>\n",
       "      <td>4.7</td>\n",
       "      <td>3.4k</td>\n",
       "      <td>Govt agency</td>\n",
       "      <td>New Delhi + 192 more</td>\n",
       "      <td>125 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Defence, Federal Agencies, Government</td>\n",
       "      <td>The principal task of the Ministry is to frame...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>28</td>\n",
       "      <td>Mahindra &amp; Mahindra</td>\n",
       "      <td>4.3</td>\n",
       "      <td>3.2k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 275 more</td>\n",
       "      <td>75 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Automobile, Machinery / Equipment, Manufacturi...</td>\n",
       "      <td>Mahindra &amp; Mahindra Limited is an Indian multi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>29</td>\n",
       "      <td>Ericsson</td>\n",
       "      <td>4.1</td>\n",
       "      <td>3.1k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Stockholm + 110 more</td>\n",
       "      <td>144 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Analytics, Research &amp; Development, MNC</td>\n",
       "      <td>Ericsson is one of the leading providers of In...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           name rating reviews       domain  \\\n",
       "0                           TCS    3.9   16.1k       Public   \n",
       "1                     Accenture    4.0   14.1k      Private   \n",
       "2                    ICICI Bank    4.1   12.7k       Public   \n",
       "3                     Cognizant    3.9   12.1k      Private   \n",
       "4                     HDFC Bank    4.0   10.9k       Public   \n",
       "5                       Infosys    3.9   10.8k       Public   \n",
       "6                           L&T    4.1   10.2k       Public   \n",
       "7                     Capgemini    3.3    9.6k      Private   \n",
       "8                 Tech Mahindra    3.5    9.1k       Public   \n",
       "9              HCL Technologies    3.7    8.7k       Public   \n",
       "10                  Tata Motors    4.1    8.3k       Public   \n",
       "11                          IBM    4.0    8.2k      Private   \n",
       "12                 Reliance jio    4.0    7.9k       Public   \n",
       "13                      Genpact    4.0    7.7k      Private   \n",
       "14                    Axis Bank    4.0    7.6k       Public   \n",
       "15  Concentrix Daksh Service...    4.0    6.5k         None   \n",
       "16                        Wipro    3.8    6.5k       Public   \n",
       "17                     Vodafone    4.3      6k       Public   \n",
       "18       HDB Financial Services    4.0    5.5k      Private   \n",
       "19                Bharti Airtel    4.2    5.5k       Public   \n",
       "20          Kotak Mahindra Bank    3.8    5.2k       Public   \n",
       "21                       Amazon    4.2    4.9k      Private   \n",
       "22                     Reliance    4.2    4.6k       Public   \n",
       "23                Bajaj Finserv    4.1    3.8k       Public   \n",
       "24                IndusInd Bank    3.6    3.7k       Public   \n",
       "25                      Samsung    4.2    3.5k      Private   \n",
       "26     Hinduja Global Solutions    3.8    3.5k       Public   \n",
       "27                  Indian Army    4.7    3.4k  Govt agency   \n",
       "28          Mahindra & Mahindra    4.3    3.2k       Public   \n",
       "29                     Ericsson    4.1    3.1k      Private   \n",
       "\n",
       "                  location      years_old employee_strength  \\\n",
       "0        Mumbai + 156 more   52 years old  10000+ employees   \n",
       "1         Dublin + 87 more   31 years old  10000+ employees   \n",
       "2        Mumbai + 724 more   26 years old  10000+ employees   \n",
       "3        Teaneck + 44 more   26 years old  10000+ employees   \n",
       "4        Mumbai + 692 more   26 years old  10000+ employees   \n",
       "5      Bangalore + 64 more   39 years old  10000+ employees   \n",
       "6        Mumbai + 333 more   82 years old  10000+ employees   \n",
       "7          Paris + 42 more   53 years old  10000+ employees   \n",
       "8          Pune + 138 more   34 years old  10000+ employees   \n",
       "9         Noida + 106 more   14 years old  10000+ employees   \n",
       "10         Pune + 336 more   75 years old  10000+ employees   \n",
       "11      New York + 69 more  109 years old  10000+ employees   \n",
       "12  Navi Mumbai + 588 more   10 years old  10000+ employees   \n",
       "13      New York + 31 more   23 years old  10000+ employees   \n",
       "14       Mumbai + 645 more   27 years old  10000+ employees   \n",
       "15     New Delhi + 36 more   21 years old  10000+ employees   \n",
       "16    Bangalore + 116 more   75 years old  10000+ employees   \n",
       "17    Berkshire + 366 more   38 years old  10000+ employees   \n",
       "18    Ahmedabad + 434 more   13 years old  10000+ employees   \n",
       "19    New Delhi + 339 more   25 years old  10000+ employees   \n",
       "20       Mumbai + 344 more   17 years old  10000+ employees   \n",
       "21   Washington + 157 more   26 years old  10000+ employees   \n",
       "22       Mumbai + 206 more   54 years old  10000+ employees   \n",
       "23         Pune + 453 more   13 years old  10000+ employees   \n",
       "24       Mumbai + 354 more   26 years old  10000+ employees   \n",
       "25        Seoul + 286 more   82 years old  10000+ employees   \n",
       "26     Bangalore + 48 more   20 years old  10000+ employees   \n",
       "27    New Delhi + 192 more  125 years old  10000+ employees   \n",
       "28       Mumbai + 275 more   75 years old  10000+ employees   \n",
       "29    Stockholm + 110 more  144 years old  10000+ employees   \n",
       "\n",
       "                                                 tags  \\\n",
       "0                Consulting, IT/ITES, MNC, Fortune500   \n",
       "1                            Consulting, IT/ITES, MNC   \n",
       "2   Banking/Insurance/Accounting, Financial Servic...   \n",
       "3                                        IT/ITES, MNC   \n",
       "4   Banking/Insurance/Accounting, Financial Servic...   \n",
       "5                            IT/ITES, Fortune500, MNC   \n",
       "6              Construction, Fortune500, Conglomerate   \n",
       "7         Computer Software, Consulting, IT/ITES, MNC   \n",
       "8                Consulting, IT/ITES, MNC, Fortune500   \n",
       "9                Consulting, IT/ITES, Fortune500, MNC   \n",
       "10         Automobile, Manufacturing, Fortune500, MNC   \n",
       "11                           Consulting, IT/ITES, MNC   \n",
       "12                                   Telecom/ISP, MNC   \n",
       "13                           Consulting, IT/ITES, MNC   \n",
       "14  Banking/Insurance/Accounting, Financial Servic...   \n",
       "15                                            BPO/KPO   \n",
       "16  Consulting, IT/ITES, MNC, Conglomerate, Fortun...   \n",
       "17                       Telecom/ISP, MNC, Fortune500   \n",
       "18                                 Financial Services   \n",
       "19                            Telecom/ISP, Fortune500   \n",
       "20  Banking/Insurance/Accounting, Financial Servic...   \n",
       "21              Consumer goods, Internet, Retail, MNC   \n",
       "22  Oil/Gas/Petro Chemicals, Power & Energy, Fortu...   \n",
       "23                     Financial Services, Fortune500   \n",
       "24  Banking/Insurance/Accounting, Financial Servic...   \n",
       "25  Consumer goods, Manufacturing, Electronics, Co...   \n",
       "26                           BPO/KPO, MNC, Fortune500   \n",
       "27              Defence, Federal Agencies, Government   \n",
       "28  Automobile, Machinery / Equipment, Manufacturi...   \n",
       "29             Analytics, Research & Development, MNC   \n",
       "\n",
       "                                                about  \n",
       "0   Tata Consultancy Services is an IT services, c...  \n",
       "1   Accenture is a leading global professional ser...  \n",
       "2   ICICI Bank is India's largest private sector b...  \n",
       "3   Cognizant (NASDAQ-100: CTSH) is one of the wor...  \n",
       "4   HDFC Bank is one of India’s leading private ba...  \n",
       "5   Infosys is a global leader in next-generation ...  \n",
       "6   Larsen & Toubro is a major technology, enginee...  \n",
       "7   A global leader in consulting, technology serv...  \n",
       "8   Tech Mahindra represents the connected world, ...  \n",
       "9   HCL Learning, a wholly owned subsidiary of HCL...  \n",
       "10  Tata Motors Limited, a USD 45 billion organisa...  \n",
       "11  At IBM, work is more than a job - it's a calli...  \n",
       "12  Make a new start. A start that will build beau...  \n",
       "13  Genpact (NYSE: G) is a global professional ser...  \n",
       "14  Axis Bank is the third largest private sector ...  \n",
       "15  A wholly owned subsidiary of SYNNEX Corporatio...  \n",
       "16  Wipro Limited (NYSE: WIT, BSE: 507685, NSE: WI...  \n",
       "17  Vodafone is a leader in technology communicati...  \n",
       "18  HDB Financial Services (HDBFS) is a leading No...  \n",
       "19  Bharti Airtel is a leading alternative provide...  \n",
       "20  In February 2003, Kotak Mahindra Finance Ltd. ...  \n",
       "21  Amazon is guided by four principles: customer ...  \n",
       "22  Reliance Industries Limited is a Fortune 500 c...  \n",
       "23  Bajaj Finserv was formed in April 2007 as a re...  \n",
       "24  IndusInd Bank is a Mumbai based scheduled priv...  \n",
       "25  Samsung Electronics is a global leader in tech...  \n",
       "26  A global leader in business process management...  \n",
       "27  The principal task of the Ministry is to frame...  \n",
       "28  Mahindra & Mahindra Limited is an Indian multi...  \n",
       "29  Ericsson is one of the leading providers of In...  "
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Scraping the whole Website!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ALL SET! LET'S PUT EVERYTHING TOGETHER AND SCRAPE THE WHOLE WEBSITE !!!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Let's not scrape the whole website here... use a python script and run it in the terminal to do so!\n",
    "# the external python script is attatched with this notebook"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scraping webpage number: 1 of 10\n",
      "success!\n",
      "time taken: 1703.35 ms\n",
      "total time elapsed: 0:00:01.705312\n",
      "\n",
      "scraping webpage number: 2 of 10\n",
      "success!\n",
      "time taken: 3859.78 ms\n",
      "total time elapsed: 0:00:05.565094\n",
      "\n",
      "scraping webpage number: 3 of 10\n",
      "success!\n",
      "time taken: 1935.15 ms\n",
      "total time elapsed: 0:00:07.500243\n",
      "\n",
      "scraping webpage number: 4 of 10\n",
      "success!\n",
      "time taken: 2529.02 ms\n",
      "total time elapsed: 0:00:10.030256\n",
      "\n",
      "scraping webpage number: 5 of 10\n",
      "success!\n",
      "time taken: 1083.29 ms\n",
      "total time elapsed: 0:00:11.113542\n",
      "\n",
      "scraping webpage number: 6 of 10\n",
      "success!\n",
      "time taken: 995.52 ms\n",
      "total time elapsed: 0:00:12.109065\n",
      "\n",
      "scraping webpage number: 7 of 10\n",
      "success!\n",
      "time taken: 1618.12 ms\n",
      "total time elapsed: 0:00:13.727190\n",
      "\n",
      "scraping webpage number: 8 of 10\n",
      "success!\n",
      "time taken: 1511.58 ms\n",
      "total time elapsed: 0:00:15.238774\n",
      "\n",
      "scraping webpage number: 9 of 10\n",
      "success!\n",
      "time taken: 1509.74 ms\n",
      "total time elapsed: 0:00:16.748513\n",
      "\n",
      "scraping webpage number: 10 of 10\n",
      "success!\n",
      "time taken: 1404.3 ms\n",
      "total time elapsed: 0:00:18.152809\n",
      "\n",
      "full website scraped successfully!\n",
      "total time taken: 0:00:18.153809\n",
      "\n"
     ]
    }
   ],
   "source": [
    "start_time = time.time()\n",
    "dataframe_final = pd.DataFrame()\n",
    "\n",
    "# 4,52,568 unique companies found / 30 per page = 15086 pages\n",
    "# total_number_of_webpages = 15086\n",
    "total_number_of_webpages = 10\n",
    "\n",
    "for page in range(1, total_number_of_webpages+1):\n",
    "    print(\"scraping webpage number: {page} of {total}\".format(page=page, total=total_number_of_webpages))\n",
    "    loop_time = time.time()\n",
    "    \n",
    "    # set page url and header\n",
    "    url = \"https://www.ambitionbox.com/list-of-companies?page={}\".format(page)\n",
    "    header = {\"User-Agent\": \"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/80.0.3987.163 Safari/537.36\"}\n",
    "    \n",
    "    # get page response from the website\n",
    "    response = requests.get(url, headers=header)\n",
    "    # time.sleep(0.1)\n",
    "    \n",
    "    # pass the page to BeautifulSoup\n",
    "    soup = BeautifulSoup(response.text, 'lxml')\n",
    "    \n",
    "    # find all the company cards from the webpage\n",
    "    company_cards = soup.find_all(\"div\", class_=\"company-content-wrapper\")\n",
    "    \n",
    "    # extract all the required data from each company card and store them in a list\n",
    "    name = []\n",
    "    rating = []\n",
    "    reviews = []\n",
    "    domain = []\n",
    "    location = []\n",
    "    years_old = []\n",
    "    employee_strength = []\n",
    "    tags = []\n",
    "    about = []\n",
    "    \n",
    "    # scrap scrap scrap!\n",
    "    for card in company_cards:\n",
    "        # 1. name\n",
    "        try:\n",
    "            name.append(card.find(\"h2\").text.strip())\n",
    "        except:\n",
    "            name.append(None)\n",
    "\n",
    "        # 2. rating\n",
    "        try:\n",
    "            rating.append(card.find(\"p\", class_=\"rating\").text.strip())\n",
    "        except:\n",
    "            rating.append(None)\n",
    "\n",
    "        # 3. reviews\n",
    "        try:\n",
    "            reviews.append(card.find(\"a\", class_=\"review-count sbold-Labels\").text.strip().replace(\" Reviews\", \"\"))\n",
    "        except:\n",
    "            reviews.append(None)\n",
    "\n",
    "        # 4. domain, 5. location, 6. years old & 7. employee strength\n",
    "        info_list = card.find_all(\"p\", class_=\"infoEntity sbold-list-header\")\n",
    "        dom = None\n",
    "        loc = None\n",
    "        old = None\n",
    "        emp = None\n",
    "        for i in range(4):\n",
    "            try:\n",
    "                if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-domain':\n",
    "                    dom = info_list[i].text.strip()\n",
    "\n",
    "                if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-pin-drop':\n",
    "                    loc = info_list[i].text.strip()\n",
    "\n",
    "                if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-access-time':\n",
    "                    old = info_list[i].text.strip()\n",
    "\n",
    "                if info_list[i].findChildren(\"i\")[0][\"class\"][0] == 'icon-supervisor-account':\n",
    "                    emp = info_list[i].text.strip()\n",
    "            except:\n",
    "                pass\n",
    "\n",
    "        domain.append(dom)\n",
    "        location.append(loc)\n",
    "        years_old.append(old)\n",
    "        employee_strength.append(emp)\n",
    "\n",
    "        # 8. tags\n",
    "        t = []\n",
    "        try:\n",
    "            for tag in card.find_all(\"a\", class_=\"ab_chip\"):\n",
    "                t.append(tag.text.strip())\n",
    "            t = ', '.join(t)\n",
    "            tags.append(t)\n",
    "        except:\n",
    "            tags.append(None)\n",
    "\n",
    "        # 9. about\n",
    "        try:\n",
    "            about.append(card.find(\"p\", class_=\"description\").text.strip())\n",
    "        except:\n",
    "            about.append(None)\n",
    "    \n",
    "    # make a dictionary containing all the data extracted\n",
    "    col_dic = {\n",
    "        \"name\": name,\n",
    "        \"rating\": rating,\n",
    "        \"reviews\": reviews,\n",
    "        \"domain\": domain,\n",
    "        \"location\": location,\n",
    "        \"years_old\": years_old,\n",
    "        \"employee_strength\": employee_strength,\n",
    "        \"tags\": tags,\n",
    "        \"about\": about\n",
    "    }\n",
    "    \n",
    "    # pass the dictionary to pandas to create a dataframe (page)\n",
    "    df = pd.DataFrame(col_dic)\n",
    "    \n",
    "    # append the dataframe to the final dataframe (the whole website)\n",
    "    dataframe_final = dataframe_final.append(df, ignore_index=True)\n",
    "    \n",
    "    # success\n",
    "    print(\"success!\")\n",
    "    print(\"time taken:\", round((time.time()-loop_time)*1000, 2), \"ms\")\n",
    "    print(\"total time elapsed:\", str(timedelta(seconds=(time.time()-start_time))))\n",
    "    print()\n",
    "\n",
    "end_time = time.time()\n",
    "print(\"full website scraped successfully!\")\n",
    "print(\"total time taken:\", str(timedelta(seconds=(end_time - start_time))))\n",
    "print()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Let's see what our dataframe looks like"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>rating</th>\n",
       "      <th>reviews</th>\n",
       "      <th>domain</th>\n",
       "      <th>location</th>\n",
       "      <th>years_old</th>\n",
       "      <th>employee_strength</th>\n",
       "      <th>tags</th>\n",
       "      <th>about</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>TCS</td>\n",
       "      <td>3.9</td>\n",
       "      <td>16.1k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 156 more</td>\n",
       "      <td>52 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC, Fortune500</td>\n",
       "      <td>Tata Consultancy Services is an IT services, c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>Accenture</td>\n",
       "      <td>4.0</td>\n",
       "      <td>14.1k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Dublin + 87 more</td>\n",
       "      <td>31 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Consulting, IT/ITES, MNC</td>\n",
       "      <td>Accenture is a leading global professional ser...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>ICICI Bank</td>\n",
       "      <td>4.1</td>\n",
       "      <td>12.7k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 724 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Servic...</td>\n",
       "      <td>ICICI Bank is India's largest private sector b...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>Cognizant</td>\n",
       "      <td>3.9</td>\n",
       "      <td>12.1k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Teaneck + 44 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>IT/ITES, MNC</td>\n",
       "      <td>Cognizant (NASDAQ-100: CTSH) is one of the wor...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>4</td>\n",
       "      <td>HDFC Bank</td>\n",
       "      <td>4.0</td>\n",
       "      <td>10.9k</td>\n",
       "      <td>Public</td>\n",
       "      <td>Mumbai + 692 more</td>\n",
       "      <td>26 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Servic...</td>\n",
       "      <td>HDFC Bank is one of India’s leading private ba...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         name rating reviews   domain           location     years_old  \\\n",
       "0         TCS    3.9   16.1k   Public  Mumbai + 156 more  52 years old   \n",
       "1   Accenture    4.0   14.1k  Private   Dublin + 87 more  31 years old   \n",
       "2  ICICI Bank    4.1   12.7k   Public  Mumbai + 724 more  26 years old   \n",
       "3   Cognizant    3.9   12.1k  Private  Teaneck + 44 more  26 years old   \n",
       "4   HDFC Bank    4.0   10.9k   Public  Mumbai + 692 more  26 years old   \n",
       "\n",
       "  employee_strength                                               tags  \\\n",
       "0  10000+ employees               Consulting, IT/ITES, MNC, Fortune500   \n",
       "1  10000+ employees                           Consulting, IT/ITES, MNC   \n",
       "2  10000+ employees  Banking/Insurance/Accounting, Financial Servic...   \n",
       "3  10000+ employees                                       IT/ITES, MNC   \n",
       "4  10000+ employees  Banking/Insurance/Accounting, Financial Servic...   \n",
       "\n",
       "                                               about  \n",
       "0  Tata Consultancy Services is an IT services, c...  \n",
       "1  Accenture is a leading global professional ser...  \n",
       "2  ICICI Bank is India's largest private sector b...  \n",
       "3  Cognizant (NASDAQ-100: CTSH) is one of the wor...  \n",
       "4  HDFC Bank is one of India’s leading private ba...  "
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe_final.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "dataframe shape (300, 9)\n",
      "\n",
      "column-wise null count\n",
      "name                 0\n",
      "rating               0\n",
      "reviews              0\n",
      "domain               9\n",
      "location             0\n",
      "years_old            1\n",
      "employee_strength    1\n",
      "tags                 0\n",
      "about                2\n",
      "dtype: int64\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# Print some statistics about the final dataframe:\n",
    "print(\"dataframe shape\", dataframe_final.shape)\n",
    "print()\n",
    "print(\"column-wise null count\")\n",
    "print(dataframe_final.isna().sum())\n",
    "print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>rating</th>\n",
       "      <th>reviews</th>\n",
       "      <th>domain</th>\n",
       "      <th>location</th>\n",
       "      <th>years_old</th>\n",
       "      <th>employee_strength</th>\n",
       "      <th>tags</th>\n",
       "      <th>about</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>count</td>\n",
       "      <td>300</td>\n",
       "      <td>300</td>\n",
       "      <td>300</td>\n",
       "      <td>291</td>\n",
       "      <td>300</td>\n",
       "      <td>299</td>\n",
       "      <td>299</td>\n",
       "      <td>300</td>\n",
       "      <td>298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>unique</td>\n",
       "      <td>300</td>\n",
       "      <td>15</td>\n",
       "      <td>143</td>\n",
       "      <td>4</td>\n",
       "      <td>287</td>\n",
       "      <td>107</td>\n",
       "      <td>3</td>\n",
       "      <td>185</td>\n",
       "      <td>298</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>top</td>\n",
       "      <td>Reliance Life Insurance</td>\n",
       "      <td>4.2</td>\n",
       "      <td>1.1k</td>\n",
       "      <td>Private</td>\n",
       "      <td>Mumbai + 67 more</td>\n",
       "      <td>20 years old</td>\n",
       "      <td>10000+ employees</td>\n",
       "      <td>Banking/Insurance/Accounting, Financial Services</td>\n",
       "      <td>Through helping other organizations mitigate r...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>freq</td>\n",
       "      <td>1</td>\n",
       "      <td>61</td>\n",
       "      <td>28</td>\n",
       "      <td>142</td>\n",
       "      <td>3</td>\n",
       "      <td>15</td>\n",
       "      <td>246</td>\n",
       "      <td>13</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                           name rating reviews   domain          location  \\\n",
       "count                       300    300     300      291               300   \n",
       "unique                      300     15     143        4               287   \n",
       "top     Reliance Life Insurance    4.2    1.1k  Private  Mumbai + 67 more   \n",
       "freq                          1     61      28      142                 3   \n",
       "\n",
       "           years_old employee_strength  \\\n",
       "count            299               299   \n",
       "unique           107                 3   \n",
       "top     20 years old  10000+ employees   \n",
       "freq              15               246   \n",
       "\n",
       "                                                    tags  \\\n",
       "count                                                300   \n",
       "unique                                               185   \n",
       "top     Banking/Insurance/Accounting, Financial Services   \n",
       "freq                                                  13   \n",
       "\n",
       "                                                    about  \n",
       "count                                                 298  \n",
       "unique                                                298  \n",
       "top     Through helping other organizations mitigate r...  \n",
       "freq                                                    1  "
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataframe_final.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exporting the dataframe into an external CSV"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "# export the data to external csv\n",
    "dataframe_final.to_csv(\"dataset/List_of_companies_in_India_ipynb_demo.csv\", encoding=\"utf-8\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}