ARWU-ranking.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "installed finish\n"
     ]
    }
   ],
   "source": [
    "import requests\n",
    "from bs4 import BeautifulSoup\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "import time\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "\n",
    "import pandas as pd\n",
    "\n",
    "print(\"installed finish\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "finish\n"
     ]
    }
   ],
   "source": [
    "from selenium.webdriver.support.ui import Select\n",
    "# test for open driver\n",
    "option = Options()\n",
    "option.add_argument(\"--disable-notifications\")\n",
    "chrome = webdriver.Chrome('./chromedriver', options=option)\n",
    "\n",
    "url = \"https://www.shanghairanking.com/rankings/arwu/2020\"\n",
    "chrome.get(url)\n",
    "soup = BeautifulSoup(chrome.page_source, 'html.parser')\n",
    "chrome.close()\n",
    "print(\"finish\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_one_score(soup, build):\n",
    "    length = 5 if build else 1\n",
    "    result_list = [[] for i in range(length)]\n",
    "\n",
    "    # for each row\n",
    "    for item in soup.select(\"tr\"):\n",
    "        td = item.select(\"td\")\n",
    "        if(len(td) == 0):\n",
    "            continue\n",
    "        row = []\n",
    "\n",
    "        try:\n",
    "            if(build):\n",
    "                # ranking\n",
    "                row.append(td[0].find(class_=\"ranking\").contents[0].lstrip().rstrip())  \n",
    "                # university name\n",
    "                try:\n",
    "                    row.append(td[1].find(class_=\"univ-name\").contents[0].lstrip().rstrip())\n",
    "                except:\n",
    "                    row.append(td[1].find(class_=\"univ-name-normal\").contents[0].lstrip().rstrip())\n",
    "                # regional rank\n",
    "                row.append(td[3].contents[0].lstrip().rstrip())\n",
    "                # total score\n",
    "                row.append(td[4].contents[0].lstrip().rstrip())\n",
    "            # item score\n",
    "            row.append(td[5].contents[0].lstrip().rstrip())\n",
    "        except:\n",
    "            if(build):\n",
    "                print(\"break error\")\n",
    "            else:\n",
    "                print(\"item error\")\n",
    "            continue\n",
    "\n",
    "        for index in range(len(result_list)):\n",
    "            result_list[index].append(row[index])\n",
    "\n",
    "    return result_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def write_to_csv(ranks, name, regional, total, scores):\n",
    "    indicators = [\"Alumni\", \"Award\", \"HiCi\", \"N&S\", \"PUB\", \"PCP\"]\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "    df[\"World Rank\"] = ranks\n",
    "    df[\"Institution\"] = name\n",
    "    df[\"National/Regional Rank\"] = regional\n",
    "    df[\"Total Score\"] = total\n",
    "\n",
    "    for index in range(len(indicators)):\n",
    "        df[indicators[index]] = scores[index]\n",
    "\n",
    "    # write file with append method\n",
    "    df.to_csv(\"ARWU-ranking.csv\", mode='a', header=False, index=False, encoding=\"utf-8\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "current page: 1\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 2\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 3\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 4\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 5\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 6\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 7\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 8\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 9\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 10\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 11\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 12\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 13\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 14\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 15\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 16\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 17\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 18\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 19\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 20\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 21\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 22\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 23\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 24\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 25\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 26\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 27\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 28\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 29\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 30\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 31\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 32\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 33\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "current page: 34\n",
      "Alumni | Award | HiCi | N&S | PUB | PCP | \n",
      "finish\n"
     ]
    }
   ],
   "source": [
    "indicators = [\"Alumni\", \"Award\", \"HiCi\", \"N&S\", \"PUB\", \"PCP\"]\n",
    "file = open(\"ARWU-ranking.csv\", 'w')\n",
    "file.write(\"World Rank,Institution,National/Regional Rank,Total Score,\")\n",
    "for index in range(len(indicators)):\n",
    "    file.write(indicators[index] + \",\")\n",
    "file.write(\"\\n\")\n",
    "file.close()\n",
    "\n",
    "option = Options()\n",
    "option.add_argument(\"--disable-notifications\")\n",
    "chrome = webdriver.Chrome('./chromedriver', options=option)\n",
    "\n",
    "url = \"https://www.shanghairanking.com/rankings/arwu/2020\"\n",
    "chrome.get(url)\n",
    "soup = BeautifulSoup(chrome.page_source, 'html.parser')\n",
    "\n",
    "page_count = 1\n",
    "while(page_count <= 34):\n",
    "    time.sleep(1)\n",
    "    print(\"current page:\", page_count)\n",
    "    ranks, name, regional, total = [], [], [], []\n",
    "    scores = [[] for i in range(6)]\n",
    "\n",
    "    # for 6 item score\n",
    "    btn_name_list = [\"Alumni\", \"Award\", \"HiCi\", \"N&S\", \"PUB\", \"PCP\"]\n",
    "\n",
    "    for index in range(6):\n",
    "        btns = chrome.find_elements_by_class_name(\"head-bg\", )\n",
    "        btns[-1].click()\n",
    "        print(indicators[index], end=\" | \")\n",
    "\n",
    "        links = chrome.find_elements_by_tag_name(\"li\")\n",
    "        for link in links:\n",
    "            if(link.text == btn_name_list[index]):\n",
    "                link.click()\n",
    "                soup = BeautifulSoup(chrome.page_source, 'lxml')\n",
    "\n",
    "                if(index == 0):\n",
    "                    result = get_one_score(soup, True)\n",
    "                    ranks += result[0]\n",
    "                    name += result[1]\n",
    "                    regional += result[2]\n",
    "                    total += result[3]\n",
    "                    scores[index] += result[4]\n",
    "                else:\n",
    "                    result = get_one_score(soup, False)\n",
    "                    scores[index] += result[0]\n",
    "\n",
    "                time.sleep(2)\n",
    "                break  # to click option button\n",
    "    \n",
    "    print()\n",
    "    # click next page\n",
    "    btns_next_page = chrome.find_elements_by_class_name(\"ant-pagination-item-link\", )\n",
    "    btns_next_page[-1].click()\n",
    "    # scroll to top\n",
    "    js=\"var action = document.documentElement.scrollTop = 0\"\n",
    "    chrome.execute_script(js)\n",
    "    time.sleep(2)\n",
    "\n",
    "    # write to file\n",
    "    write_to_csv(ranks, name, regional, total, scores)\n",
    "\n",
    "    page_count += 1\n",
    "\n",
    "chrome.close()\n",
    "print(\"finish\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}