01-download-pages.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "49a97dbb",
   "metadata": {},
   "source": [
    "# Download EPSA 2019 authors, sessions and abstracts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d68c4fed",
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.common.exceptions import TimeoutException\n",
    "import os\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "7fbd2773",
   "metadata": {},
   "source": [
    "## Create download folders"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d0c6534c",
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in [\"authors\", \"abstracts\", \"sessions\"]:\n",
    "    os.makedirs(\"html/\" + i, exist_ok = True)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "99073854",
   "metadata": {},
   "source": [
    "## Initiate Web driver"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "d2f12e12",
   "metadata": {},
   "outputs": [],
   "source": [
    "driver = webdriver.Chrome('/Applications/chromedriver')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "04e4cefe",
   "metadata": {},
   "source": [
    "## Download authors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b5221085",
   "metadata": {},
   "source": [
    "### Check whether downloading authors is already done"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cd47603f",
   "metadata": {},
   "source": [
    "The notebook will go through (and download) all author listings if any of them is missing."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "86f93143",
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "All authors already downloaded.\n"
     ]
    }
   ],
   "source": [
    "download_authors = True\n",
    "\n",
    "files = [f for f in os.listdir(\"html/authors\") if re.match(r'.*\\.html', f)]\n",
    "if len(files) > 0:\n",
    "    \n",
    "    # extract numbers from downloaded authors lists filenames\n",
    "    first_id = []\n",
    "    last_id = []\n",
    "    max_id = []\n",
    "    \n",
    "    for i in files:\n",
    "        n = re.findall(r'\\d+', i)\n",
    "        # store 'x to y out of z' values\n",
    "        first_id.append(int(n[0]))\n",
    "        last_id.append(int(n[1]))\n",
    "        max_id.append(int(n[2]))\n",
    "\n",
    "    max_id = set(max_id)\n",
    "    assert len(max_id) == 1, \"Malformed author count.\"\n",
    "\n",
    "    # determine whether we are already done\n",
    "    if 1 in set(first_id) and max(set(last_id)) == max(max_id):\n",
    "        print('All authors already downloaded.')\n",
    "        download_authors = False"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "576ada23",
   "metadata": {},
   "source": [
    "### If needed, loop through author listings and download"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "5916c97a",
   "metadata": {},
   "outputs": [],
   "source": [
    "if download_authors:\n",
    "    \n",
    "    driver.get(\"https://app.oxfordabstracts.com/events/772/program-app/authors\")\n",
    "\n",
    "    i = 0\n",
    "\n",
    "    # loop until results counter says otherwise\n",
    "    while download_authors:\n",
    "\n",
    "        i += 1\n",
    "\n",
    "        # wait for main content to finish loading\n",
    "        try:\n",
    "            WebDriverWait(driver, 7.5).until(EC.presence_of_element_located((By.TAG_NAME, 'h2')))\n",
    "        except TimeoutException:\n",
    "            print(\"Download of author pages failed at iteration \" + str(i) +\"\\n\")\n",
    "            raise SystemExit(\"Could not download all author pages\")\n",
    "\n",
    "        # extract results string\n",
    "        r = driver.find_element_by_xpath(\".//span[@class='results__count']\").text\n",
    "        print(str(i).rjust(3) + \" Dowloaded authors \" + r)\n",
    "\n",
    "        # save\n",
    "        f = \"html/authors/authors_\" + r.replace(\" \", \"_\") + \".html\"\n",
    "        with open(f, \"w\") as file:\n",
    "            file.write(driver.page_source)\n",
    "\n",
    "        # check if we reached the end\n",
    "        n = re.findall(r'\\d+', r)\n",
    "        if int(n[1]) == int(n[2]):\n",
    "            break\n",
    "\n",
    "        # if not, find first 'Next' button, click it, and loop\n",
    "        b = driver.find_element_by_xpath('.//button[@class=\"program__button\"][contains(., \"Next\")]')\n",
    "        b.click()\n",
    "\n",
    "    print(\"done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d40381ca",
   "metadata": {},
   "source": [
    "## Find links to sessions and abstracts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "9a1669ca",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Parsing 66 author pages...\n"
     ]
    }
   ],
   "source": [
    "session_urls = []\n",
    "abstract_urls = []\n",
    "\n",
    "files = [f for f in os.listdir(\"html/authors\") if re.match(r'.*\\.html', f)]\n",
    "print(\"Parsing \" + str(len(files)) + \" author pages...\")\n",
    "\n",
    "for i in sorted(files):\n",
    "    # load\n",
    "    driver.get(\"file://\" + os.path.abspath(\"html/authors/\" + i))\n",
    "    \n",
    "    # find links to sessions\n",
    "    r = driver.find_elements_by_xpath(\".//a[contains(@href, 'session')]\")\n",
    "    for j in r:\n",
    "        session_urls.append(j.get_attribute(\"href\"))\n",
    "    \n",
    "    # find links to abstracts\n",
    "    r = driver.find_elements_by_xpath(\".//a[contains(@href, 'submission')]\")\n",
    "    for j in r:\n",
    "        abstract_urls.append(j.get_attribute(\"href\"))"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ce31493",
   "metadata": {},
   "source": [
    "## Download sessions (panels)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8a6bf812",
   "metadata": {},
   "source": [
    "Session IDs are always 4 digits."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "721194d2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading 186 sessions...\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "session_urls = set(session_urls)\n",
    "print(\"Downloading \" + str(len(session_urls)) + \" sessions...\")\n",
    "\n",
    "for i in sorted(session_urls):\n",
    "    u = i.replace(\"file://\", \"https://app.oxfordabstracts.com\")\n",
    "    f = \"html/sessions/session_\" + re.search(r'\\d{4}', i).group(0) + \".html\"\n",
    "    if not(os.path.exists(f)):\n",
    "        \n",
    "        print(\"Downloading session \" + u)\n",
    "        \n",
    "        # load\n",
    "        driver.get(u)\n",
    "        \n",
    "        # wait for main content to finish loading\n",
    "        try:\n",
    "            WebDriverWait(driver, 7.5).until(EC.presence_of_element_located((By.TAG_NAME, 'h1')))\n",
    "        except TimeoutException:\n",
    "            print(\"Download of session failed at URL \" + u +\"\\n\")\n",
    "            raise SystemExit(\"Could not download all session pages\")\n",
    "\n",
    "        # save\n",
    "        with open(f, \"w\") as file:\n",
    "            file.write(driver.page_source)\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "731e9c5e",
   "metadata": {},
   "source": [
    "## Download abstracts"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "43edb329",
   "metadata": {},
   "source": [
    "Abstracts IDs are variable-length: 5-6 digits."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "651bb2e3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading 802 abstracts...\n",
      "done\n"
     ]
    }
   ],
   "source": [
    "abstract_urls = set(abstract_urls)\n",
    "\n",
    "print(\"Downloading \" + str(len(abstract_urls)) + \" abstracts...\")\n",
    "\n",
    "for i in abstract_urls:\n",
    "    u = i.replace(\"file://\", \"https://app.oxfordabstracts.com\")\n",
    "    f = \"html/abstracts/abstract_\" + re.search(r'\\d{5,6}', i).group(0) + \".html\"\n",
    "    if not(os.path.exists(f)):\n",
    "        \n",
    "        print(\"Downloading abstract \" + u)\n",
    "        \n",
    "        # load\n",
    "        driver.get(u)\n",
    "        \n",
    "        # wait for main content to finish loading\n",
    "        try:\n",
    "            WebDriverWait(driver, 7.5).until(EC.presence_of_element_located((By.TAG_NAME, 'h1')))\n",
    "        except TimeoutException:\n",
    "            print(\"Download of abstract failed at URL \" + u +\"\\n\")\n",
    "            raise SystemExit(\"Could not download all abstract pages\")\n",
    "\n",
    "        # save\n",
    "        with open(f, \"w\") as file:\n",
    "            file.write(driver.page_source)\n",
    "\n",
    "print(\"done\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4246e37d",
   "metadata": {},
   "source": [
    "Have a nice day."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}