1-scraping_subreddits.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 1. Imports and Getting Set Up"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [],
   "source": [
    "# imports\n",
    "import pandas as pd\n",
    "import requests\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 193,
   "metadata": {},
   "outputs": [],
   "source": [
    "# obscures inputs and does not echo them back\n",
    "import getpass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "metadata": {},
   "outputs": [
    {
     "name": "stdin",
     "output_type": "stream",
     "text": [
      " ······················\n",
      " ······························\n",
      " ····\n",
      " ·······\n",
      " ···········\n"
     ]
    }
   ],
   "source": [
    "# this is to keep credentials secret when entering\n",
    "\n",
    "client_id = getpass.getpass() #alphanumeric string provided under \"personal use script\"\n",
    "client_secret =  getpass.getpass() #alphanumeric string provided as \"secret\"\n",
    "user_agent =  getpass.getpass() #the name of your application\n",
    "username =  getpass.getpass() #your reddit username\n",
    "password =  getpass.getpass() #your reddit password"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "metadata": {},
   "outputs": [],
   "source": [
    "# retrieving the access token\n",
    "\n",
    "auth = requests.auth.HTTPBasicAuth(client_id, client_secret)\n",
    "\n",
    "data = {\n",
    "    'grant_type': 'password',\n",
    "    'username': username,\n",
    "    'password': password\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<Response [200]>\n"
     ]
    }
   ],
   "source": [
    "# Header for application\n",
    "headers = {'User-Agent': 'dsi1113/0.0.1'}\n",
    "\n",
    "res = requests.post(\n",
    "    'https://www.reddit.com/api/v1/access_token',\n",
    "    auth=auth,\n",
    "    data=data,\n",
    "    headers=headers)\n",
    "\n",
    "# ensuring it works\n",
    "print(res)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'access_token': 'eyJhbGciOiJSUzI1NiIsImtpZCI6IlNIQTI1NjpzS3dsMnlsV0VtMjVmcXhwTU40cWY4MXE2OWFFdWFyMnpLMUdhVGxjdWNZIiwidHlwIjoiSldUIn0.eyJzdWIiOiJ1c2VyIiwiZXhwIjoxNzA0NDk1ODYwLjU3MzE1OSwiaWF0IjoxNzA0NDA5NDYwLjU3MzE1OSwianRpIjoiaGVHaTNDNTNQcFM0UHptSHM1em1RY2ItelFxVWRnIiwiY2lkIjoiXzFsMlFPdjRaUk9jYldwT3FGVjk2USIsImxpZCI6InQyX3I5cGdsZDV3ZyIsImFpZCI6InQyX3I5cGdsZDV3ZyIsImxjYSI6MTcwNDI0MzQ5NzQ1Miwic2NwIjoiZUp5S1Z0SlNpZ1VFQUFEX193TnpBU2MiLCJmbG8iOjl9.E4LwJxMpC5M1zsH2SDwXFRd7FrQYKNZZ-kMdZA_PsYSgNKyDiBGwIm-4F9TNbDzVcZLCWCMq-mjVtJGxZXjtAZHCv3mQrP9oRQQNCj8QcOBf41QktKa40bJOoWAkzyR9Iz6b4wZHrq8oyDq6vUMNj1W0jH16wbF-iFiTDCEmTXdncSI4g9MLMxqVQFOPs5KyUWm3ezpk204cycteiq7Sa8Ueu_G6ZoZtMB0te5pSZJ8UkfuuSSOi4gX8M96WSpRyV4YFOttqoeHFdoEhy5oPzfuQRu8gKs_KI4bjBDxoWrtQNO1Usa-SbqiUdZUR_5XOtI6sXWKV4dR5qz9WQOt8EQ',\n",
       " 'token_type': 'bearer',\n",
       " 'expires_in': 86400,\n",
       " 'scope': '*'}"
      ]
     },
     "execution_count": 198,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# this will cause an error if something did not work\n",
    "res.json()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 199,
   "metadata": {},
   "outputs": [],
   "source": [
    "# retrieve access token\n",
    "token = res.json()['access_token']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 200,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "True"
      ]
     },
     "execution_count": 200,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# add token to headers\n",
    "headers['Authorization'] = f'bearer {token}'\n",
    "\n",
    "requests.get('https://oauth.reddit.com/api/v1/me', headers=headers).status_code == 200"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 2. Testing to see if we can start scraping"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 201,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'kind': 't3',\n",
       " 'data': {'approved_at_utc': None,\n",
       "  'subreddit': 'ADHD',\n",
       "  'selftext': \"Hey y'all,\\n\\nOnce the rules vacation is over, we're going to be collecting feedback about the current rules vacation and the state of the sub. Specifically, we're interested in feedback only on the rules that were suspended. We aren't sure what that's going to look like just yet, but it will likely consist of a set of ratings asking how you feel about each rule and the state of the sub, plus space for written comments. The questionnaire should be posted early January.\\n\\nUntil then, keep the following questions in mind:\\n\\n* What do you like about the current state of the sub? What don't you like?\\n* Are there any rules you want to see removed or modified? Are there any you think we should keep?\\n* How do you feel about the volume of posts? Is it overwhelming or just right?\\n* How is the balance of discussion posts to memes and other lower-effort content? Are important things getting drowned out?\\n* How much misinformation, toxic positivity, pseudoscience, and alternative medicine discussion is there? Is it going unchallenged? Or are there people correcting it?\\n\\nWe're announcing this ahead of time so people have time to gather their thoughts without feeling pressured.\",\n",
       "  'author_fullname': 't2_45e8k',\n",
       "  'saved': False,\n",
       "  'mod_reason_title': None,\n",
       "  'gilded': 0,\n",
       "  'clicked': False,\n",
       "  'title': 'Feedback on Rules Vacation and State of /r/adhd',\n",
       "  'link_flair_richtext': [],\n",
       "  'subreddit_name_prefixed': 'r/ADHD',\n",
       "  'hidden': False,\n",
       "  'pwls': None,\n",
       "  'link_flair_css_class': 'PSA',\n",
       "  'downs': 0,\n",
       "  'thumbnail_height': None,\n",
       "  'top_awarded_type': None,\n",
       "  'hide_score': False,\n",
       "  'name': 't3_18ukx9o',\n",
       "  'quarantine': False,\n",
       "  'link_flair_text_color': 'light',\n",
       "  'upvote_ratio': 0.91,\n",
       "  'author_flair_background_color': None,\n",
       "  'subreddit_type': 'public',\n",
       "  'ups': 16,\n",
       "  'total_awards_received': 0,\n",
       "  'media_embed': {},\n",
       "  'thumbnail_width': None,\n",
       "  'author_flair_template_id': None,\n",
       "  'is_original_content': False,\n",
       "  'user_reports': [],\n",
       "  'secure_media': None,\n",
       "  'is_reddit_media_domain': False,\n",
       "  'is_meta': False,\n",
       "  'category': None,\n",
       "  'secure_media_embed': {},\n",
       "  'link_flair_text': 'Mod Announcement',\n",
       "  'can_mod_post': False,\n",
       "  'score': 16,\n",
       "  'approved_by': None,\n",
       "  'is_created_from_ads_ui': False,\n",
       "  'author_premium': False,\n",
       "  'thumbnail': 'self',\n",
       "  'edited': 1703967045.0,\n",
       "  'author_flair_css_class': None,\n",
       "  'author_flair_richtext': [],\n",
       "  'gildings': {},\n",
       "  'content_categories': None,\n",
       "  'is_self': True,\n",
       "  'mod_note': None,\n",
       "  'created': 1703957815.0,\n",
       "  'link_flair_type': 'text',\n",
       "  'wls': None,\n",
       "  'removed_by_category': None,\n",
       "  'banned_by': None,\n",
       "  'author_flair_type': 'text',\n",
       "  'domain': 'self.ADHD',\n",
       "  'allow_live_comments': False,\n",
       "  'selftext_html': '&lt;!-- SC_OFF --&gt;&lt;div class=\"md\"&gt;&lt;p&gt;Hey y&amp;#39;all,&lt;/p&gt;\\n\\n&lt;p&gt;Once the rules vacation is over, we&amp;#39;re going to be collecting feedback about the current rules vacation and the state of the sub. Specifically, we&amp;#39;re interested in feedback only on the rules that were suspended. We aren&amp;#39;t sure what that&amp;#39;s going to look like just yet, but it will likely consist of a set of ratings asking how you feel about each rule and the state of the sub, plus space for written comments. The questionnaire should be posted early January.&lt;/p&gt;\\n\\n&lt;p&gt;Until then, keep the following questions in mind:&lt;/p&gt;\\n\\n&lt;ul&gt;\\n&lt;li&gt;What do you like about the current state of the sub? What don&amp;#39;t you like?&lt;/li&gt;\\n&lt;li&gt;Are there any rules you want to see removed or modified? Are there any you think we should keep?&lt;/li&gt;\\n&lt;li&gt;How do you feel about the volume of posts? Is it overwhelming or just right?&lt;/li&gt;\\n&lt;li&gt;How is the balance of discussion posts to memes and other lower-effort content? Are important things getting drowned out?&lt;/li&gt;\\n&lt;li&gt;How much misinformation, toxic positivity, pseudoscience, and alternative medicine discussion is there? Is it going unchallenged? Or are there people correcting it?&lt;/li&gt;\\n&lt;/ul&gt;\\n\\n&lt;p&gt;We&amp;#39;re announcing this ahead of time so people have time to gather their thoughts without feeling pressured.&lt;/p&gt;\\n&lt;/div&gt;&lt;!-- SC_ON --&gt;',\n",
       "  'likes': None,\n",
       "  'suggested_sort': None,\n",
       "  'banned_at_utc': None,\n",
       "  'view_count': None,\n",
       "  'archived': False,\n",
       "  'no_follow': False,\n",
       "  'is_crosspostable': True,\n",
       "  'pinned': False,\n",
       "  'over_18': False,\n",
       "  'all_awardings': [],\n",
       "  'awarders': [],\n",
       "  'media_only': False,\n",
       "  'link_flair_template_id': '43181b5c-d4ff-11e3-b1b3-12313d18e5cd',\n",
       "  'can_gild': False,\n",
       "  'spoiler': False,\n",
       "  'locked': True,\n",
       "  'author_flair_text': None,\n",
       "  'treatment_tags': [],\n",
       "  'visited': False,\n",
       "  'removed_by': None,\n",
       "  'num_reports': None,\n",
       "  'distinguished': 'moderator',\n",
       "  'subreddit_id': 't5_2qnwb',\n",
       "  'author_is_blocked': False,\n",
       "  'mod_reason_by': None,\n",
       "  'removal_reason': None,\n",
       "  'link_flair_background_color': '#0aa18f',\n",
       "  'id': '18ukx9o',\n",
       "  'is_robot_indexable': True,\n",
       "  'report_reasons': None,\n",
       "  'author': 'nerdshark',\n",
       "  'discussion_type': None,\n",
       "  'num_comments': 0,\n",
       "  'send_replies': True,\n",
       "  'whitelist_status': None,\n",
       "  'contest_mode': False,\n",
       "  'mod_reports': [],\n",
       "  'author_patreon_flair': False,\n",
       "  'author_flair_text_color': None,\n",
       "  'permalink': '/r/ADHD/comments/18ukx9o/feedback_on_rules_vacation_and_state_of_radhd/',\n",
       "  'parent_whitelist_status': None,\n",
       "  'stickied': True,\n",
       "  'url': 'https://www.reddit.com/r/ADHD/comments/18ukx9o/feedback_on_rules_vacation_and_state_of_radhd/',\n",
       "  'subreddit_subscribers': 1769290,\n",
       "  'created_utc': 1703957815.0,\n",
       "  'num_crossposts': 0,\n",
       "  'media': None,\n",
       "  'is_video': False}}"
      ]
     },
     "execution_count": 201,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Let's see if we've connected appropriately and if we can start scraping data\n",
    "base_url = 'https://oauth.reddit.com/r/'\n",
    "subreddit1 = 'ADHD'\n",
    "\n",
    "# make limit 100 instead of defult 25 (MAX IS 100)\n",
    "params = {\n",
    "    'limit': 100\n",
    "   # 'after': <-- will be important for getting the 'next' posts\n",
    "}\n",
    "\n",
    "res1 = requests.get(base_url+subreddit1, \n",
    "                   headers=headers,\n",
    "                   params= params)\n",
    "\n",
    "# 'data' houses all the response info\n",
    "# 'after', 'before', and 'children'\n",
    "\n",
    "# let's just get the first (index 0) post to avoid a long output\n",
    "res1.json()['data']['children'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The above request works as anticipated and we were able to pull the first post. <br><br>\n",
    "Next, let's write a function that will scrape the most recent ~1000 posts from a given subreddit for us. We will call the function using an input (the names of the subreddits), concatenate the dataframes, and then store the resulting dataframe and save it off as a csv file."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 3. Scraping the Subreddits and Storing Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to return a dataframe of posts, title, sub, up/downvotes given the name of a subreddit\n",
    "def subreddit_scrape(name):\n",
    "    \n",
    "    # empty lists (this is where we will store data)\n",
    "    posts = []\n",
    "    title = []\n",
    "    subreddit = []\n",
    "    ups = []\n",
    "    downs = []\n",
    "    \n",
    "    # create separate pieces for URL\n",
    "    base_url = 'https://oauth.reddit.com/r/'\n",
    "    subreddit1 = name # this is where our input will be called in\n",
    "    endpoint = '/new' # this specifies we are looking at the new posts (can change to 'hot' or 'top')\n",
    "    \n",
    "    \n",
    "    params = {\n",
    "        'limit': 100,\n",
    "        'after': ''} # 'after' will be updated in the 1st for loop\n",
    "\n",
    "    for n in range(1,12): # pull data 11 times\n",
    "        # pull the data using our url, headers, and params\n",
    "        data = requests.get(base_url+subreddit1+endpoint, \n",
    "                           headers=headers,\n",
    "                           params=params)\n",
    "        \n",
    "        # this is our 'counter' --> updates 'after' for every batch of posts we pull, so the next pull will look at the next batch of posts\n",
    "        params.update({'after': data.json()['data']['after']}) \n",
    "        \n",
    "        # for the current pull of data, iterate thru and store the following data in the lists above\n",
    "        for i in range(0, len(data.json()['data']['children'])):\n",
    "            posts.append(data.json()['data']['children'][i]['data']['selftext'])\n",
    "            title.append(data.json()['data']['children'][i]['data']['title'])\n",
    "            subreddit.append(data.json()['data']['children'][i]['data']['subreddit'])\n",
    "            ups.append(data.json()['data']['children'][i]['data']['ups'])\n",
    "            downs.append(data.json()['data']['children'][i]['data']['downs'])\n",
    "\n",
    "    # return a dataframe\n",
    "    return pd.DataFrame({'post': posts, 'title': title, 'subreddit': subreddit, 'upvotes': ups, 'downvotes': downs}).drop_duplicates()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 212,
   "metadata": {},
   "outputs": [],
   "source": [
    "# store scraped data\n",
    "r_adhd = subreddit_scrape('ADHD')\n",
    "r_autism = subreddit_scrape('autism')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 211,
   "metadata": {},
   "outputs": [],
   "source": [
    "# concatenate both dataframes and save off as csv\n",
    "\n",
    "pd.concat([r_adhd,r_autism]).to_csv('./data/scraped_reddit_data.csv')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}