nsgl-or-catalogscrape_selenium.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "a0b1bd01-b476-4d91-bd80-e745b476b252",
   "metadata": {},
   "source": [
    "# National Sea Grant Library (NGSL) catalog metadata scrape - Oregon edition\n",
    "\n",
    "The intent here is to get a feel for how many of the NSGL catalog records for California have an associated PDF. The NGSL is closing it's physical space at the University of Rhode Island. I am trying to determine how much of the physical collection still needs to be digitized, and if I should accession the physical collection for safekeeping.\n",
    "\n",
    "----"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "285e3bbc-c7ef-4677-a9b1-f9b43b4e0640",
   "metadata": {},
   "outputs": [],
   "source": [
    "# import libraries we need \n",
    "\n",
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "import pandas as pd\n",
    "import re\n",
    "import urllib.parse\n",
    "import time\n",
    "import datetime\n",
    "\n",
    "# If the Chrome browser is already on your system, you should just need to do\n",
    "# pip instal selenium\n",
    "# pip install webdriver-manager\n",
    "# to install the necessary packages.\n",
    "\n",
    "from selenium import webdriver\n",
    "from webdriver_manager.chrome import ChromeDriverManager\n",
    "from pandas.io.html import read_html\n",
    "from selenium.webdriver.common.action_chains import ActionChains\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from urllib.parse import urlparse\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.common.exceptions import NoSuchWindowException\n"
   ]
  },
  {
   "cell_type": "raw",
   "id": "fe24109e-f247-4763-a0d0-b7adbe0318ae",
   "metadata": {
    "tags": []
   },
   "source": [
    "### This section will save an HTML file for each page of 100 catalog search results. The search was:\n",
    "\n",
    "Word(s): [“oregon” and “sea grant”]\n",
    "\n",
    "which resulted in 4,016 results at 100 results per page."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fee675c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This should fire up the headless Chrome browser in another window\n",
    "driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "driver.implicitly_wait(2) # Crude way to ensure the page has (mostly) loaded before doing anything else."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e2c5c827-4ab3-4e3b-9992-7cad77d613f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "total_pages = 41\n",
    "current_page = 1\n",
    "\n",
    "# url = 'https://eos.ucs.uri.edu/EOSWebOPAC/OPAC/Search/AdvancedSearch.aspx?TaskCode=737107&TitleListPageSize=100&CatLevel0Value=&CatLevel1Value=&CatLevel2Value=&CatLevel3Value=&CatLevel4Value='\n",
    "url= 'https://eos.ucs.uri.edu/EOSWebOPAC/OPAC/Search/AdvancedSearch.aspx?TaskCode=820219&TitleListPageSize=100&CatLevel0Value=&CatLevel1Value=&CatLevel2Value=&CatLevel3Value=&CatLevel4Value='\n",
    "#grab = requests.get(url)\n",
    "#soup = BeautifulSoup(grab.text, 'html.parser')\n",
    "driver.get(url)\n",
    "\n",
    "for current_page in range(1,total_pages+1):\n",
    "\n",
    "    grab = driver.page_source\n",
    "\n",
    "    # Zero-pad the search page number (2 digits minimum) when writing the page source\n",
    "    with open(f\"or_search_results_page_{current_page:02}.html\", \"w\") as outfile:\n",
    "        outfile.write(grab)\n",
    "\n",
    "    next_button = driver.find_element_by_name(\"ctl00$webopacContentHolder$SearchTitleListControl$titleListNav1$arrowRight\")\n",
    "    if next_button is not None:\n",
    "        time.sleep(3) # to be polite\n",
    "        next_button.click()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3b4f0f6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gracefully shut down the headless browser\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "70267001-864d-4bb2-bf4b-91fd3f0329f1",
   "metadata": {},
   "source": [
    "---\n",
    "## Now sort through the scraped results pages to obtain an identifier for each item. With that in hand, scrape additional metadata and any associated files for each record. \n",
    "\n",
    "---"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88bc931e-1dc8-4fc2-9065-b71812bbeead",
   "metadata": {},
   "outputs": [],
   "source": [
    "#io is used for opening and writing files\n",
    "import io\n",
    "\n",
    "#glob is used to find all the pathnames matching a specified pattern (here, all text files)\n",
    "import glob\n",
    "\n",
    "#os is used to navigate your folder directories (e.g. change folders to where you files are stored)\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e03aeba3-4bc3-4561-9533-1eb9e2be4395",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the file directory here\n",
    "filedirectory = '/Users/thalassa/github/nsgl/pages-searchResults_OR'\n",
    "\n",
    "#Change the working directory to the one you just defined\n",
    "os.chdir(filedirectory)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "902c3164-2459-4525-bbd7-8515087d6b26",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(os.listdir(filedirectory))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "83fc93c3-a451-4a8c-a748-fa81c0daa718",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Sort all the files in the directory you specified above, alphabetically.\n",
    "\n",
    "start = datetime.datetime.utcnow()\n",
    "\n",
    "#For each of those files...\n",
    "for filename in sorted(os.listdir(filedirectory)):\n",
    "    #If the filename ends with .html (i.e. if it's actually a text files)\n",
    "    if filename.endswith('.html'):\n",
    "        #The file name of the output file adds _data to the end of the file name of the input file\n",
    "        outfilename = filename.replace('.html', '_data.txt')\n",
    "        #Open the infput filename\n",
    "        with open(filename, 'r') as f:\n",
    "            #Create and open the output filename\n",
    "            with open(outfilename, 'w') as out:\n",
    "                soup = BeautifulSoup(f, \"html.parser\")\n",
    "                records = str(soup.find_all(href=re.compile(\"javascript:ViewNewCompleteDisplayRecord\")))\n",
    "                ids = re.findall(r'[A-Z]*\\|[0-9]*\\|[0-9]*\\|[0-9]*',records)\n",
    "                for element in ids:\n",
    "                    out.write(element + \"\\n\")\n",
    "                out.close()\n",
    "                \n",
    "end = datetime.datetime.utcnow()\n",
    "print(f\"Finished at {end}, total time {(end-start).seconds / 60.} minutes.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "730171ce-b0db-45e6-8045-2c3103097ac9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# merge all of the recrods ids into one file\n",
    "read_files = glob.glob(\"*.txt\")\n",
    "\n",
    "with open(\"seagreant-or-ids.txt\", \"wb\") as outfile:\n",
    "    for f in read_files:\n",
    "        with open(f, \"rb\") as infile:\n",
    "            outfile.write(infile.read())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d815a99a",
   "metadata": {},
   "source": [
    "## Now we want to parse the IDs to get the information we need to build the URLs to fetch the data\n",
    "\n",
    "E.g., EOSMAIN|5393927|10|2696314\n",
    "\n",
    "Is a link like this: https://eos.ucs.uri.edu/EOSWebOPAC/OPAC/Details/Record.aspx?BibCode=EOSMAIN%7C5393927%7C10%7C2696314"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c8cfdb78",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the file directory here\n",
    "filedirectory = '/Users/thalassa/github/nsgl'\n",
    "\n",
    "#Change the working directory to the one you just defined\n",
    "os.chdir(filedirectory)\n",
    "\n",
    "# Open the text file with the item IDs in it, create a list\n",
    "f = open(\"seagrant-or-ids.txt\",'r')\n",
    "ids = f.read().split(\"\\n\")\n",
    "print(ids[0:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d6d4be70",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Open text file and create a dataframe\n",
    "data = pd.read_csv(\"seagrant-or-ids.txt\",sep=\"|\",names=[\"eosmain\",\"id1\",\"id2\",\"id3\"])\n",
    "data.head"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3e653002",
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a list of URLs with the ID info as another column in the 'data' dataframe\n",
    "data['url'] = 'https://eos.ucs.uri.edu/EOSWebOPAC/OPAC/Details/Record.aspx?BibCode=EOSMAIN%7C' + data['id1'].astype(str) + '%7C' + data['id2'].astype(str) + '%7C' + data['id3'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "35d515cf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Show one URL \n",
    "data['url'][0]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1c08ef2b",
   "metadata": {},
   "source": [
    "## Now we can launch the ChromeDriver, use the links we generated in the previous steps to visit each catalog record, scrape the catalog metadata and PDF files from the records.\n",
    "\n",
    "The steps are:\n",
    "- load catalog record from list of URLs\n",
    "- find metadata table, read it, save to CSV file\n",
    "- see if PDF links exist\n",
    "- grab pdf links, iterate through and download each one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "49f4694a",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This should fire up the headless Chrome browser in another window\n",
    "driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "driver.implicitly_wait(2) # Crude way to ensure the page has (mostly) loaded before doing anything else."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4fdc768b",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = data['url'][1]\n",
    "driver.get(url)\n",
    "canclick = driver.find_element_by_xpath('//*[@id=\"MediaListRepeater_ctl00_MediaHyperLink\"]').get_attribute('onclick')\n",
    "print(canclick)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d9a16171",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Loop through number of URLs in data DataFrame\n",
    "total_items = len(data)\n",
    "current_item = 1821\n",
    "\n",
    "#set up filenames for CSV files\n",
    "outfilename = data['id1'].astype(str) + \"_\" + data['id2'].astype(str) + \"_\" + data['id3'].astype(str) + \".csv\"\n",
    "\n",
    "for current_item in range(current_item,total_items+1):\n",
    "    item_url = data['url'][current_item]\n",
    "\n",
    "    driver.get(item_url)\n",
    "    \n",
    "    try: \n",
    "        table = driver.find_element_by_xpath('//*[@id=\"wrapper\"]/div[2]/div/div/table/tbody/tr/td[2]')\n",
    "    except NoSuchElementException as exception:\n",
    "        # pass\n",
    "        # print(\"Element Exception Skipped\")\n",
    "        continue \n",
    "        \n",
    "    table_html = table.get_attribute('innerHTML')\n",
    "    df = read_html(table_html)[0]\n",
    "        \n",
    "    # Save metadata to a CSV file, only the relevant columns\n",
    "    df.to_csv(outfilename[current_item],',',columns=[1,2],header=[\"MetadataField\",\"Metadata\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b35d6ee1-b429-4193-b3b9-be2e63fa2211",
   "metadata": {},
   "outputs": [],
   "source": [
    "# If the loop above hits an error, you can run this chunk to see what record you are on, update the starting record in the loop, \n",
    "# and get it going again. You may have to restart the ChromeDriver \n",
    "\n",
    "print(current_item)\n",
    "#print(data[277])\n",
    "print(data['url'][current_item])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4510671-ddfd-4c85-9ffb-c9a14305da5f",
   "metadata": {},
   "source": [
    "I used the code below to try to download PDFs from item records in the NSGL catalog. It failed alot b/c their catalog is ... dated. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7071d23-a46f-4af3-8b91-362f2fb7b765",
   "metadata": {},
   "outputs": [],
   "source": [
    "# move up into previous code chunk if you want to download the PDFs again\n",
    "# Download PDF(s) if available\n",
    "\n",
    "    canclick = driver.find_element_by_xpath('//*[@id=\"MediaListRepeater_ctl00_MediaHyperLink\"]').get_attribute('onclick')\n",
    "    if (canclick==\"TrackMediaLinkUsage\"):\n",
    "\n",
    "<a href=\"/seagrant_Linked_Documents/oresu/WR-15-001%20Edwards%20(Cone)%20R-S-18-PD%20(poster)%20.pdf\" id=\"MediaListRepeater_ctl00_MediaHyperLink\" class=\"trigger\" title=\"\" target=\"_blank\" onclick=\"TrackMediaLinkUsage('2','view PDF','0','10593614','10593699','1' );\" mediacode=\"10593699\"><img src=\"/EOSWebOPAC/Images/mediatype90X90-2.png\" id=\"MediaListRepeater_ctl00_ImageHolder\" onmouseout=\"ViewDetailMouseOut(this)\" width=\"90\" data-original=\"\" class=\"loading90 thumbNailImages lazy\" height=\"90\" onmouseover=\"ViewDetail('/EOSWebOPAC/Images/mediatype350X350-2.png','view PDF','',' ','0',$(this),10593699)\" style=\"z-index:1;\" alt=\"view PDF\"></a>\n",
    "\n",
    "    # pdf_relative_link is\n",
    "    # '/SEAGRANT_Linked_Documents/scu/USC%20Sea%20Grant%20Newsletter_%c2%a0April%202020.pdf'\n",
    "    grab = driver.page_source\n",
    "    pdf_relative_link = re.findall(\"seagrant_Linked_Documents\\S*.pdf\",grab)\n",
    "    pdflink = 'https://eos.ucs.uri.edu/' + pdf_relative_link\n",
    "    pdf_filename = pdf_relative_link.split('/')[-1]\n",
    "    with open(pdf_filename, 'wb') as f:\n",
    "        f.write(response.content)\n",
    "        f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "385fb07b-0021-43a1-88ab-434cf782c769",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This should fire up the headless Chrome browser in another window\n",
    "driver = webdriver.Chrome(ChromeDriverManager().install())\n",
    "driver.implicitly_wait(2) # Crude way to ensure the page has (mostly) loaded before doing anything else."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ff157aa3-8a70-4e4a-b403-d012143a0383",
   "metadata": {},
   "source": [
    "Now we need to click through all the items for the first 100 search results, then click on the \"next page\" arrow, then resume clicking through items until we hit item 200, then click \"next page\", and so on until we've gone through 62 pages of results (100 items per page). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e77bf01d-a6d7-4c3d-8bf1-da9b973865da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# I pulled this chunk of text out to another cell so that I can start on whatever \"current_item\" I need to (due to errors in the catalog, etc.) \n",
    "total_items = 6109\n",
    "current_item = 1\n",
    "\n",
    "# Search results URL vvv\n",
    "# NOTE this URL may not work. In that case, go to \"Search the Catalog\" and do a Word(s) search for “california and sea and grant”. Then you can run this cell and it will work.\n",
    "url = 'https://eos.ucs.uri.edu/EOSWebOPAC/OPAC/Search/AdvancedSearch.aspx?TaskCode=739469&TitleListPageSize=100&CatLevel0Value=&CatLevel1Value=&CatLevel2Value=&CatLevel3Value=&CatLevel4Value='\n",
    "driver.get(url)\n",
    "\n",
    "# click on first record in the list of 100 results on first page. After this you will be clicking through item pages. \n",
    "# Comment out this line if you are restarting loop in the middle somewhere. You must be on the record URL you want to start with. \n",
    "first_record = driver.find_element_by_xpath('//*[@id=\"ctl00_webopacContentHolder_SearchTitleListControl_MainRepeater_ctl01_DetailRepeater_ctl01_DetailRow\"]/td/a').click()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38957f4d-78de-44f5-91fc-2c6293c3b6da",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This loops through 6109 results, one item at a time. There is logic for what to do when you hit the bottom of a results page.\n",
    "for current_item in range(current_item,total_items+1):\n",
    "    \n",
    "    grab = driver.page_source\n",
    "\n",
    "    # Zero-pad the search page number (4 digits minimum) when writing the page source\n",
    "    with open(f\"search_results_item_{current_item:04}.html\", \"w\") as outfile:\n",
    "        outfile.write(grab)\n",
    "        \n",
    "    # click the \"down arrow\" to go to the next record in the list of 100 results on this page.\n",
    "    # We need to check to see if the down arrow will function - it won't on the last record.\n",
    "    # Outputs of this attribute check will return \"GetNextTitle();\" when there are more results to show, \n",
    "    # or \"return false;\" when the button does not work (at record 100 on the page)\n",
    "        canclick = driver.find_element_by_xpath('//*[@id=\"ctl00_ctl00_webopacContentHolder_RecordPager_imgGetNext\"]').get_attribute('onclick')\n",
    "    if (canclick==\"GetNextTitle();\"):\n",
    "        \n",
    "        # click the \"down arrow\" to go to the next record in the list of 100 results on this page.\n",
    "        next_record = driver.find_element_by_xpath('//*[@id=\"ctl00_ctl00_webopacContentHolder_RecordPager_imgGetNext\"]')\n",
    "        time.sleep(3) # to be polite\n",
    "        next_record.click()   \n",
    "    \n",
    "    # After the scraper gets to record 100, we need to turn the page by doing a few clicks.\n",
    "    else:\n",
    "        # click the \"Records\" drop-down:\n",
    "        driver.find_element_by_xpath('//*[@id=\"ctl00_ctl00_webopacContentHolder_RecordPager_imgExpandNav\"]').click()\n",
    "\n",
    "        # and then click the \"next\" button:\n",
    "        driver.find_element_by_xpath('//*[@id=\"ctl00_ctl00_webopacContentHolder_RecordPager_NextPage\"]').click()\n",
    "\n",
    "        # and then click the first item in the drop-down list, which I think always has the same ID:\n",
    "        driver.find_element_by_xpath('//*[@id=\"ctl00_ctl00_webopacContentHolder_RecordPager_RecordRepeater_ctl01_ItemAnchor\"]').click()\n",
    "\n",
    "        continue # to the top of the loop for the 100 records on the page\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aedebd68-453e-454a-8f04-2468257a526d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "fc48a2b4-47eb-4539-a4d5-ab099e0b8b13",
   "metadata": {},
   "source": [
    "## Literally everything below here is me randomly trying sh*t to see what sticks."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24402212-7a7f-4317-8477-13c393e80b38",
   "metadata": {},
   "outputs": [],
   "source": [
    "url = 'https://eos.ucs.uri.edu/EOSWebOPAC/OPAC/Search/AdvancedSearch.aspx?TaskCode=737107&TitleListPageSize=100&CatLevel0Value=&CatLevel1Value=&CatLevel2Value=&CatLevel3Value=&CatLevel4Value='\n",
    "grab = requests.get(url)\n",
    "soup = BeautifulSoup(grab.text, 'html.parser') # parse text from the html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "44268c1b-1f99-456c-ae98-29ae9a7fea82",
   "metadata": {},
   "outputs": [],
   "source": [
    "TABLE_SELECTOR = \"table.TableCellPadding2Px:nth-child(3) > tbody:nth-child(1)\"\n",
    "\n",
    "#ctl00_webopacContentHolder_SearchTitleListControl_titleListNav1_arrowRight\n",
    "document.querySelector(\"#ctl00_webopacContentHolder_SearchTitleListControl_titleListNav1_arrowRight\") \n",
    "\n",
    "#ctl00_webopacContentHolder_SearchTitleListControl_MainRepeater_ctl01_DetailColumn > table:nth-child(1)\n",
    "\n",
    "#ctl00_webopacContentHolder_SearchTitleListControl_MainRepeater_ctl01_DetailColumn > table:nth-child(1) > tbody:nth-child(1)\n",
    "\n",
    "#ctl00_webopacContentHolder_SearchTitleListControl_MainRepeater_ctl01_DetailRepeater_ctl01_DetailRow "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2eb6b996-f027-4107-a0f6-4db3c7736d52",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_test = soup.find_all('table', class_='DefaultTable')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fd654103-eedb-420d-9c22-b9159dbf3873",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(table_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "95db4127-6248-47ee-8d97-63a68582c5b6",
   "metadata": {},
   "outputs": [],
   "source": [
    "table_test = soup.find_all('tr', class_='HorDisplayAltRow')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "515ec852-865e-4479-bc86-3aa55ea28bec",
   "metadata": {},
   "outputs": [],
   "source": [
    "test = soup.find_all('tbody')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8811426e-0ea3-49a8-9512-06b48edede33",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b7437fe-8178-4d63-a280-dd9fc1a60655",
   "metadata": {},
   "outputs": [],
   "source": [
    "print(soup.re_compile(\"seagrant_Linked_Documents\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "940d8b15-b13b-452e-a257-ce373f6d4483",
   "metadata": {},
   "outputs": [],
   "source": [
    "urls = []\n",
    "for link in soup.find_all('a'):\n",
    "    print(link.get('href'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66664305-dd9e-4c43-8a46-4664d76f836c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# opening a file in write mode\n",
    "f = open(\"test1.txt\", \"w\")\n",
    "# traverse paragraphs from soup\n",
    "for link in soup.find_all(\"a\"):\n",
    "   data = link.get('href')\n",
    "   f.write(data)\n",
    "   f.write(\"\\n\")\n",
    " \n",
    "f.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6432d34e-0172-4e27-b08d-4c47c988d601",
   "metadata": {},
   "outputs": [],
   "source": [
    "# traverse paragraphs from soup\n",
    "for link in soup.find_all(\"a\"):\n",
    "   data = link.get('href')\n",
    "   print(data)\n",
    "   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a648b411-8d9c-4cb4-8209-44a8aefdf3ee",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}