Data_scraper.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "TGx1Cc2UDsF1"
      },
      "source": [
        "## Order of colab reference notebooks:\n",
        "1. Reddit Data Scrapper.ipynb \\\n",
        "2. Pipeline_for_BOW.ipynb  \\\n",
        "3. dataset_classification.ipynb \\\n",
        "4. Toxic_Span_Detection_BiLSTM+CRF_model.ipynb \\"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Hx6Ec4-tFV9g"
      },
      "source": [
        "# Dataset Collection"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {
        "id": "-z0BAxfuFQfe"
      },
      "source": [
        "Installing necessary libraries"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "J_Z-XE7oDr5V",
        "outputId": "2deed73a-f2f0-422c-e268-527cc5840274"
      },
      "outputs": [],
      "source": [
        "# !pip install praw\n",
        "# !pip install asyncpraw\n",
        "# import praw\n",
        "import tqdm\n",
        "import asyncpraw\n",
        "import re\n",
        "# from praw.models import MoreComments\n",
        "import pandas as pd\n",
        "import asyncio"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Fetching news subreddit data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "dh8qDW_iNaCz"
      },
      "outputs": [],
      "source": [
        "reddit = asyncpraw.Reddit(client_id='q6HKrPg90SJKqevoa6i8Tw', client_secret='wlS_eetUTdF7oddrOLdABpBkQW14EQ', user_agent='nlp-toxic')\n",
        "news_subreddit = await reddit.subreddit('news')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 380
        },
        "id": "1QQfd9-3NaAV",
        "outputId": "bf1937ed-995a-4f31-e71a-c30870fb0bb0"
      },
      "outputs": [],
      "source": [
        "posts = []\n",
        "comments_reddit = []\n",
        "\n",
        "async for post in news_subreddit.hot(limit=200):\n",
        "    posts.append([post.id])\n",
        "posts = pd.DataFrame(posts, columns=[\"id\"])"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Fetching the comments from the news subreddit and preprocessing "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "Tzc4_bdnNZ9m",
        "outputId": "dd98ac7c-6516-4583-f9e4-c131134ad702"
      },
      "outputs": [],
      "source": [
        "emoji_pattern = re.compile(\"[\"\n",
        "        u\"\\U0001F600-\\U0001F64F\"  # emoticons\n",
        "        u\"\\U0001F300-\\U0001F5FF\"  # symbols & pictographs\n",
        "        u\"\\U0001F680-\\U0001F6FF\"  # transport & map symbols\n",
        "        u\"\\U0001F1E0-\\U0001F1FF\"  # flags (iOS)\n",
        "                           \"]+\", flags=re.UNICODE)\n",
        "\n",
        "\n",
        "comments_reddit=[]\n",
        "for subid in posts[\"id\"][0:1]:\n",
        "    submission = await reddit.submission(subid)\n",
        "    print(subid)\n",
        "    await submission.comments.replace_more(limit=None)\n",
        "    for comment in submission.comments.list():\n",
        "        sentencing = comment.body\n",
        "        #Removing spaces and new lines in a single comment\n",
        "        sentencing = sentencing.replace('\\n','')\n",
        "        sentencing = sentencing.replace('\\t','')\n",
        "        sentencing = sentencing.strip()\n",
        "        #Removing any urls\n",
        "        # sentencing = sentencing.replace('https\\S+|www.\\S+', '')\n",
        "        sentencing = re.sub(r'https\\S+|www.\\S+','',sentencing)\n",
        "        #1 or more white spaces with single whitespace\n",
        "        sentencing = re.sub(r'\\s+', ' ', sentencing)\n",
        "        # sentencing = emoji_pattern.sub(r'',sentencing)\n",
        "        comments_reddit.append(sentencing)"
      ]
    },
    {
      "attachments": {},
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "Save the comments to a csv file"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "comments_subreddit = pd.DataFrame(comments_reddit, columns=[\"comments\"])\n",
        "comments_subreddit.to_csv(\"news_comments.csv\")"
      ]
    }
  ],
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.16"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}