Crawl_data_new.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Crawl_data_new.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "source": [
        "# Crawler \n",
        "\n",
        "input: \n",
        "*   the link of the project guttenburg under the category of \"stories\"\n",
        "\n",
        "output: \n",
        "\n",
        "*   custom_dataset.txt (the file containing all the text for each book)\n",
        "*   book_id.txt (a file contaning the id's from each book)\n",
        "*   test_txt.txt (a file contaning a paragraph of 100 words from each book)\n",
        "\n"
      ],
      "metadata": {
        "id": "NHvzQKmYM3at"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "import bs4 as bs\n",
        "import urllib.request\n",
        "import re\n",
        "import requests\n",
        "from bs4 import BeautifulSoup\n",
        "import pandas as pd\n",
        "from google.colab import drive"
      ],
      "metadata": {
        "id": "GdGnvcRwXDLM"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "drive.mount('/drive')"
      ],
      "metadata": {
        "id": "vE4_00VzvGZp",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3be362a7-9e62-4ceb-ef40-f1dca9f46043"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Mounted at /drive\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# function used to return all the text inside a url\n",
        "def get_text(soup):\n",
        "    for script in soup(\"script\"):\n",
        "        script.decompose()\n",
        "    text = soup.get_text()\n",
        "    return text"
      ],
      "metadata": {
        "id": "8ORNa5tTYINF"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#story_counter = 0\n",
        "index = 1 # counter for the number of the stories we want to crawl\n",
        "story_index = [] # here we save the id of stories we have already crawled\n",
        "\n",
        "# f will be the file containing the text for all the books(each line of the txt will refer to one book)\n",
        "# g will be a txt file that will contain the id's of the books we will crawl\n",
        "# t will be the test file, containing a paragraph of text from each book\n",
        "f = open(\"custom_dataset.txt\", \"w\")\n",
        "g = open(\"book_id.txt\", \"w\")\n",
        "t = open(\"test_txt.txt\", \"w\")\n",
        "\n",
        "# we crawled 970 total books\n",
        "while index <= 970:\n",
        "  if (index%50==0):\n",
        "    # this is used so we know how many books we have added so far (prints once every 50 books)\n",
        "    print(\"=========\")\n",
        "    print(\"index =\", index)\n",
        "  \n",
        "  # link will be the link of the page Guttenberg from which we want to crawl the books\n",
        "  link = \"https://www.gutenberg.org/ebooks/search/?query=stories&submit_search=Go%21&start_index=\"+str(index)\n",
        "  agent = {\"User-Agent\":'Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36'}\n",
        "\n",
        "  # we create a request to get on the links inside the initial page we mentioned above (link)\n",
        "  res = requests.get(link, headers=agent)\n",
        "  soup = BeautifulSoup(res.text, \"html.parser\")\n",
        "  href = soup.find_all('a',href=True)\n",
        "  \n",
        "\n",
        "  for i in href:\n",
        "    # for every link we check its length since the url of each book had a certain structure\n",
        "    new_link = i['href']\n",
        "    if len(new_link) > 8 and len(new_link) <= 13:\n",
        "      # from each link, we extract a, which will be an integer (id of the story)\n",
        "      a = list(new_link)\n",
        "      a = \"\".join(a[8:])\n",
        "\n",
        "      if int(a) not in story_index:\n",
        "        story_index.append(int(a))\n",
        "\n",
        "        # we create the new url by adding the story id in the end and we request to \n",
        "        # get all the links inside the above url\n",
        "        first = \"https://www.gutenberg.org/ebooks/\"+a\n",
        "        res1 = requests.get(first, headers=agent)\n",
        "        soup1 = BeautifulSoup(res1.text, \"html.parser\")\n",
        "        href1 = soup1.find_all('a',href=True)\n",
        "\n",
        "\n",
        "        for j in href1:\n",
        "          # for each one of the new links, we check its suffix, since the url containing \n",
        "          # the plain text of each book had a suffix of either \".txt\" or \".txt.utf-8\"\n",
        "          j = j['href']\n",
        "          if j[-4:] == \".txt\" or j[-10:] == \".txt.utf-8\":\n",
        "\n",
        "            # we access the new link with the above suffixes and we request to extract all the text inside it\n",
        "            second = \"https://www.gutenberg.org\"+j\n",
        "            res2 = requests.get(second, headers=agent)\n",
        "            soup2 = BeautifulSoup(res2.text, \"html.parser\")\n",
        "            story = get_text(soup2)\n",
        "\n",
        "            # here we check if the book we are examining is english or not by checking \n",
        "            # the plain text of the book in case it contans the phrase \"Language: English\"\n",
        "            # we did this since not all book pages provided us with metadata about the language\n",
        "            is_english = re.search(r'\\b(Language: )\\b', story)\n",
        "            if is_english != None:\n",
        "              check = is_english.end()\n",
        "            else: \n",
        "              check = 10\n",
        "            \n",
        "\n",
        "            if story[check:check+7] == \"English\":\n",
        "              # if we are examining an english story, we cut a phrase of 100 words and \n",
        "              # put it inside the test txt and we also remove annecesery text from the start\n",
        "              # and from the end since every book started after the phrase:\n",
        "              # \"START OF THE PROJECT GUTENBERG EBOOK\" and every book ended before\n",
        "              # \"END OF THE PROJECT GUTENBERG EBOOK\"\n",
        "              story = story.split()\n",
        "              get_for_test = int(0.6*len(story)) \n",
        "              test_story = \" \".join(story[get_for_test:get_for_test+100])\n",
        "              story = \" \".join(story[:get_for_test] + story[get_for_test+100:])\n",
        "\n",
        "              #remove from the start\n",
        "              text_to_be_removed2 = re.search(r'\\b(START OF THE PROJECT GUTENBERG EBOOK)\\b', story)\n",
        "              if text_to_be_removed2 == None:\n",
        "                text_to_be_removed2 = re.search(r'\\b(START OF THIS PROJECT GUTENBERG EBOOK)\\b', story)\n",
        "              if text_to_be_removed2 != None:\n",
        "                story = story[text_to_be_removed2.start() + 1000:]\n",
        "              else:\n",
        "                story = story[2000:]\n",
        "\n",
        "              #remove from the end\n",
        "              text_to_be_removed = re.search(r'\\b(END OF THE PROJECT GUTENBERG EBOOK)\\b', story)\n",
        "              if text_to_be_removed == None:\n",
        "                text_to_be_removed = re.search(r'\\b(END OF THIS PROJECT GUTENBERG EBOOK)\\b', story)\n",
        "              if text_to_be_removed == None:\n",
        "                text_to_be_removed = re.search(r'\\b(This file should be named)\\b', story)\n",
        "              if text_to_be_removed != None:\n",
        "                story = story[:text_to_be_removed.start()]\n",
        "              else:\n",
        "                story = story[:len(story) - 15000]\n",
        "                \n",
        "              # we update our txt files with the new informations we crawled\n",
        "              t.write(test_story+\"\\n\")\n",
        "              f.write(story+\"\\n\")\n",
        "              g.write(str(a)+\"\\n\")\n",
        "            else:\n",
        "              # we print the id of the book in case it was non english in order to\n",
        "              # know we didnt false remove any\n",
        "              print(a)\n",
        "  index += 1\n",
        "t.close()\n",
        "f.close()\n",
        "g.close()"
      ],
      "metadata": {
        "id": "vY7swgDqXqvt"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}