1-data-scraping.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1. Data Scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import re\n",
    "import time\n",
    "import json\n",
    "\n",
    "import requests as rq\n",
    "import bs4 as bs4"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "queries = ['machine+learning', 'data+science', 'kaggle']\n",
    "url = \"https://www.youtube.com/results?search_query={query}&sp=CAI%253D&p={page}\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=1\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=2\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=3\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=4\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=5\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=6\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=7\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=8\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=9\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=10\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=11\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=12\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=13\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=14\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=15\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=16\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=17\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=18\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=19\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=20\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=21\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=22\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=23\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=24\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=25\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=26\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=27\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=28\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=29\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=30\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=31\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=32\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=33\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=34\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=35\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=36\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=37\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=38\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=39\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=40\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=41\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=42\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=43\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=44\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=45\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=46\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=47\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=48\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=49\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=50\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=51\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=52\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=53\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=54\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=55\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=56\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=57\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=58\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=59\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=60\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=61\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=62\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=63\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=64\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=65\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=66\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=67\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=68\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=69\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=70\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=71\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=72\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=73\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=74\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=75\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=76\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=77\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=78\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=79\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=80\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=81\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=82\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=83\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=84\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=85\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=86\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=87\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=88\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=89\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=90\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=91\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=92\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=93\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=94\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=95\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=96\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=97\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=98\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=99\n",
      "https://www.youtube.com/results?search_query=machine+learning&sp=CAI%253D&p=100\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=1\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=2\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=3\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=4\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=5\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=6\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=7\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=8\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=9\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=10\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=11\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=12\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=13\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=14\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=15\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=16\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=17\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=18\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=19\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=20\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=21\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=22\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=23\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=24\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=25\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=26\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=27\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=28\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=29\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=30\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=31\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=32\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=33\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=34\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=35\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=36\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=37\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=38\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=39\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=40\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=41\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=42\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=43\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=44\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=45\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=46\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=47\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=48\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=49\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=50\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=51\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=52\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=53\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=54\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=55\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=56\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=57\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=58\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=59\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=60\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=61\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=62\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=63\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=64\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=65\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=66\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=67\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=68\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=69\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=70\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=71\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=72\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=73\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=74\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=75\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=76\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=77\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=78\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=79\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=80\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=81\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=82\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=83\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=84\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=85\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=86\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=87\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=88\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=89\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=90\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=91\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=92\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=93\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=94\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=95\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=96\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=97\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=98\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=99\n",
      "https://www.youtube.com/results?search_query=data+science&sp=CAI%253D&p=100\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=1\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=2\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=3\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=4\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=5\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=6\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=7\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=8\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=9\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=10\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=11\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=12\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=13\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=14\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=15\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=16\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=17\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=18\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=19\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=20\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=21\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=22\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=23\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=24\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=25\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=26\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=27\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=28\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=29\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=30\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=31\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=32\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=33\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=34\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=35\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=36\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=37\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=38\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=39\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=40\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=41\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=42\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=43\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=44\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=45\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=46\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=47\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=48\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=49\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=50\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=51\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=52\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=53\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=54\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=55\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=56\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=57\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=58\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=59\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=60\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=61\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=62\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=63\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=64\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=65\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=66\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=67\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=68\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=69\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=70\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=71\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=72\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=73\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=74\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=75\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=76\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=77\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=78\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=79\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=80\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=81\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=82\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=83\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=84\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=85\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=86\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=87\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=88\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=89\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=90\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=91\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=92\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=93\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=94\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=95\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=96\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=97\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=98\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=99\n",
      "https://www.youtube.com/results?search_query=kaggle&sp=CAI%253D&p=100\n"
     ]
    }
   ],
   "source": [
    "for query in queries:\n",
    "    for page in range(1,101):\n",
    "        urll = url.format(query=query, page=page)\n",
    "        print(urll)\n",
    "        response = rq.get(urll)\n",
    "        \n",
    "        with open(\"./raw_data/{}_{}.html\".format(query,page), 'w+') as output:\n",
    "            output.write(response.text)\n",
    "        time.sleep(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "for query in queries:\n",
    "    for page in range(1,101):\n",
    "        with open(\"./raw_data/{}_{}.html\".format(query,page), 'r+') as inp:\n",
    "            page_html = inp.read()\n",
    "            parsed = bs4.BeautifulSoup(page_html)\n",
    "            tags = parsed.findAll(\"a\")\n",
    "            \n",
    "            for e in tags:\n",
    "                if e.has_attr(\"aria-describedby\"):\n",
    "                    link = e['href']\n",
    "                    title = e['title']\n",
    "                    with open(\"parsed_videos.json\", 'a+') as output:\n",
    "                        data = {\"link\": link, \"title\": title, \"query\": query}\n",
    "                        output.write(\"{}\\n\".format(json.dumps(data)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.read_json(\"parsed_videos.json\", lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>link</th>\n",
       "      <th>title</th>\n",
       "      <th>query</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>/watch?v=djTkaQ4SIpo</td>\n",
       "      <td>How To Use AI and Machine Learning With Your M...</td>\n",
       "      <td>machine+learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>/watch?v=tLaLfaIJf-Y</td>\n",
       "      <td>What's the Difference between Training and Tes...</td>\n",
       "      <td>machine+learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>/watch?v=k1l551lfT2g</td>\n",
       "      <td>A Conversation on Machine Learning with Cerner</td>\n",
       "      <td>machine+learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>/watch?v=JmliaDSZL1Q</td>\n",
       "      <td>A Conversation on Machine Learning with Gallup</td>\n",
       "      <td>machine+learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>/watch?v=7YDSsmG_e9s</td>\n",
       "      <td>A Conversation on Machine Learning with Cerner...</td>\n",
       "      <td>machine+learning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1688</th>\n",
       "      <td>/watch?v=VgVcm331jN8</td>\n",
       "      <td>AXA Attracts Top Data Scientists with a Data S...</td>\n",
       "      <td>kaggle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1689</th>\n",
       "      <td>/watch?v=HGr1yQV3Um0</td>\n",
       "      <td>Allstate Purchase Prediction Challenge on Kaggle</td>\n",
       "      <td>kaggle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1690</th>\n",
       "      <td>/watch?v=zwa9uc0LpXo</td>\n",
       "      <td>Krishna Sankar: Data Wrangling for Kaggle Data...</td>\n",
       "      <td>kaggle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1691</th>\n",
       "      <td>/watch?v=OShGuf7QeJY</td>\n",
       "      <td>Technology Pioneer 2014⎪Anthony Goldbloom⎪Kaggle</td>\n",
       "      <td>kaggle</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1692</th>\n",
       "      <td>/watch?v=OKOlO9nIHUE</td>\n",
       "      <td>Jeremy Howard of Kaggle speaks about Big Data ...</td>\n",
       "      <td>kaggle</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1693 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                      link                                              title  \\\n",
       "0     /watch?v=djTkaQ4SIpo  How To Use AI and Machine Learning With Your M...   \n",
       "1     /watch?v=tLaLfaIJf-Y  What's the Difference between Training and Tes...   \n",
       "2     /watch?v=k1l551lfT2g     A Conversation on Machine Learning with Cerner   \n",
       "3     /watch?v=JmliaDSZL1Q     A Conversation on Machine Learning with Gallup   \n",
       "4     /watch?v=7YDSsmG_e9s  A Conversation on Machine Learning with Cerner...   \n",
       "...                    ...                                                ...   \n",
       "1688  /watch?v=VgVcm331jN8  AXA Attracts Top Data Scientists with a Data S...   \n",
       "1689  /watch?v=HGr1yQV3Um0   Allstate Purchase Prediction Challenge on Kaggle   \n",
       "1690  /watch?v=zwa9uc0LpXo  Krishna Sankar: Data Wrangling for Kaggle Data...   \n",
       "1691  /watch?v=OShGuf7QeJY   Technology Pioneer 2014⎪Anthony Goldbloom⎪Kaggle   \n",
       "1692  /watch?v=OKOlO9nIHUE  Jeremy Howard of Kaggle speaks about Big Data ...   \n",
       "\n",
       "                 query  \n",
       "0     machine+learning  \n",
       "1     machine+learning  \n",
       "2     machine+learning  \n",
       "3     machine+learning  \n",
       "4     machine+learning  \n",
       "...                ...  \n",
       "1688            kaggle  \n",
       "1689            kaggle  \n",
       "1690            kaggle  \n",
       "1691            kaggle  \n",
       "1692            kaggle  \n",
       "\n",
       "[1693 rows x 3 columns]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}