clawscript.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "PYTHONIOENCODING='utf-8'\n",
    "from selenium import webdriver\n",
    "from selenium.common.exceptions import NoSuchElementException\n",
    "from selenium.webdriver.common.by import By\n",
    "from bs4 import BeautifulSoup\n",
    "import urllib, sys, os\n",
    "import re\n",
    "import time\n",
    "import codecs\n",
    "import json\n",
    "import traceback\n",
    "import datetime\n",
    "import pickle as pk\n",
    "chromedriver = \"./chromedriver\"\n",
    "os.environ[\"webdriver.chrome.driver\"] = chromedriver\n",
    "driver = webdriver.Chrome(chromedriver)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "hdr = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36',}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "opener = urllib.request.build_opener()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_year_info():\n",
    "    req = urllib.request.Request('https://www.aclweb.org/anthology/sigs/sigdat/', headers=hdr)\n",
    "    response = opener.open(req)\n",
    "    bsObj = BeautifulSoup(response, \"html\")\n",
    "    rows = bsObj.find_all('div', { \"class\" : \"row\" })\n",
    "    all_volume = []\n",
    "    for row in rows:\n",
    "        year = row.h4.text\n",
    "        procedding = row.a.text\n",
    "        url = 'https://www.aclweb.org' + row.a['href']\n",
    "        if int(year) > 2009:\n",
    "            continue\n",
    "        all_volume.append([int(year), procedding, url])\n",
    "    return all_volume"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_volumes = get_year_info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_volumes = sorted(all_volumes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[[1993,\n",
       "  'VERY LARGE CORPORA: ACADEMIC AND INDUSTRIAL PERSPECTIVES',\n",
       "  'https://www.aclweb.org/anthology/volumes/very-large-corpora-academic-and-industrial-perspectives/'],\n",
       " [1994,\n",
       "  'Second Workshop on Very Large Corpora (WVLC-2)',\n",
       "  'https://www.aclweb.org'],\n",
       " [1995,\n",
       "  'Third Workshop on Very Large Corpora',\n",
       "  'https://www.aclweb.org/anthology/volumes/third-workshop-on-very-large-corpora/'],\n",
       " [1996,\n",
       "  'Fourth Workshop on Very Large Corpora',\n",
       "  'https://www.aclweb.org/anthology/volumes/fourth-workshop-on-very-large-corpora/'],\n",
       " [1997,\n",
       "  'Fifth Workshop on Very Large Corpora',\n",
       "  'https://www.aclweb.org/anthology/volumes/fifth-workshop-on-very-large-corpora/'],\n",
       " [1998,\n",
       "  'Sixth Workshop on Very Large Corpora',\n",
       "  'https://www.aclweb.org/anthology/volumes/sixth-workshop-on-very-large-corpora/'],\n",
       " [1999,\n",
       "  '1999 Joint SIGDAT Conference on Empirical Methods in Natural Language Processing and Very Large Corpora',\n",
       "  'https://www.aclweb.org/anthology/volumes/1999-joint-sigdat-conference-on-empirical-methods-in-natural-language-processing-and-very-large-corpora/'],\n",
       " [2000,\n",
       "  '2000 Joint SIGDAT Conference on Empirical Methods in Natural Language Processing and Very Large Corpora',\n",
       "  'https://www.aclweb.org/anthology/volumes/2000-joint-sigdat-conference-on-empirical-methods-in-natural-language-processing-and-very-large-corpora/'],\n",
       " [2001,\n",
       "  'Proceedings of the 2001 Conference on Empirical Methods in Natural Language Processing',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2001-conference-on-empirical-methods-in-natural-language-processing/'],\n",
       " [2002,\n",
       "  'Proceedings of the 2002 Conference on Empirical Methods in Natural Language Processing (EMNLP 2002)',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2002-conference-on-empirical-methods-in-natural-language-processing-emnlp-2002/'],\n",
       " [2003,\n",
       "  'Proceedings of the 2003 Conference on Empirical Methods in Natural Language Processing',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2003-conference-on-empirical-methods-in-natural-language-processing/'],\n",
       " [2004,\n",
       "  'Proceedings of the 2004 Conference on Empirical Methods in Natural Language Processing',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2004-conference-on-empirical-methods-in-natural-language-processing/'],\n",
       " [2005,\n",
       "  'Proceedings of Human Language Technology Conference and Conference on Empirical Methods in Natural Language Processing',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-human-language-technology-conference-and-conference-on-empirical-methods-in-natural-language-processing/'],\n",
       " [2006,\n",
       "  'Proceedings of the 2006 Conference on Empirical Methods in Natural Language Processing',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2006-conference-on-empirical-methods-in-natural-language-processing/'],\n",
       " [2007,\n",
       "  'Proceedings of the 2007 Joint Conference on Empirical Methods in Natural Language Processing and Computational Natural Language Learning (EMNLP-CoNLL)',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2007-joint-conference-on-empirical-methods-in-natural-language-processing-and-computational-natural-language-learning-emnlp-conll/'],\n",
       " [2008,\n",
       "  'Proceedings of the 2008 Conference on Empirical Methods in Natural Language Processing',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2008-conference-on-empirical-methods-in-natural-language-processing/'],\n",
       " [2009,\n",
       "  'Proceedings of the 2009 Conference on Empirical Methods in Natural Language Processing',\n",
       "  'https://www.aclweb.org/anthology/volumes/proceedings-of-the-2009-conference-on-empirical-methods-in-natural-language-processing/']]"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_volumes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_all_cite(article_url):\n",
    "    driver.get(article_url)\n",
    "    atl_page = driver.page_source.encode('utf-8')\n",
    "    atl_obj = BeautifulSoup(atl_page)\n",
    "    total_citation = int(atl_obj.find('div', {'class':'gs_ri'}).find('div', {'class':'gs_fl'}).find_all('a')[2].text.split()[-1])\n",
    "#     cited_section = atl_obj.find('div', {'class':'gs_ri'}).find('div', {'class':'gs_fl'}).find_all('a')[2]\n",
    "#     total_citation = int(cited_section.text.split()[-1])\n",
    "#     cited_info_url = 'https://scholar.google.com.sg'+cited_section['href']\n",
    "    time.sleep(30)\n",
    "    return total_citation\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_volume_info(year_url):\n",
    "    vlm_req = urllib.request.Request(year_url, headers=hdr)\n",
    "    vlm_page = opener.open(vlm_req)\n",
    "    vlm_obj = BeautifulSoup(vlm_page)\n",
    "    articles = vlm_obj.find('div', {'id': 'main-container'}).section.find_all('p')[2:]\n",
    "    articles_info = []\n",
    "    for article in articles:\n",
    "        atl_url = article.span.a['href']\n",
    "        title = article.strong.text\n",
    "        googlescl = 'https://scholar.google.com.sg/scholar?hl=en&as_sdt=0%2C5&q='+title.replace(' ', '+')+'&btnG='\n",
    "        atl_ctn = get_all_cite(googlescl)\n",
    "        atl_authors = ', '.join([itm.text for itm in article.find_all('a') if 'anthology/people/' in itm['href']])\n",
    "        articles_info.append([title, atl_url, atl_authors, atl_ctn])\n",
    "        print([title, atl_url, atl_authors, atl_ctn])\n",
    "    return articles_info"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "year 1993\n"
     ]
    },
    {
     "ename": "AttributeError",
     "evalue": "'NoneType' object has no attribute 'find'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-56-42cb557fd513>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mitm\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m==\u001b[0m\u001b[0;36m1994\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0;32mcontinue\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m     \u001b[0mcnt\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mget_volume_info\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mitm\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      7\u001b[0m     \u001b[0mall_info\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mitm\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcnt\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      8\u001b[0m     \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'%d.pkl'\u001b[0m\u001b[0;34m%\u001b[0m\u001b[0mitm\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'wb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mfilout\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-55-9650c8b97944>\u001b[0m in \u001b[0;36mget_volume_info\u001b[0;34m(year_url)\u001b[0m\n\u001b[1;32m      9\u001b[0m         \u001b[0mtitle\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0marticle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstrong\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m         \u001b[0mgooglescl\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'https://scholar.google.com.sg/scholar?hl=en&as_sdt=0%2C5&q='\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreplace\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m' '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'+'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m+\u001b[0m\u001b[0;34m'&btnG='\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 11\u001b[0;31m         \u001b[0matl_ctn\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_all_cite\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mgooglescl\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     12\u001b[0m         \u001b[0matl_authors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m', '\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjoin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mitm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mitm\u001b[0m \u001b[0;32min\u001b[0m \u001b[0marticle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'a'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mif\u001b[0m \u001b[0;34m'anthology/people/'\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mitm\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'href'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     13\u001b[0m         \u001b[0marticles_info\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtitle\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0matl_url\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0matl_authors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0matl_ctn\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-54-9b886e5da2f9>\u001b[0m in \u001b[0;36mget_all_cite\u001b[0;34m(article_url)\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0matl_page\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdriver\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpage_source\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mencode\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'utf-8'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m     \u001b[0matl_obj\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mBeautifulSoup\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0matl_page\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m     \u001b[0mtotal_citation\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0matl_obj\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'div'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'class'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m'gs_ri'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'div'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m'class'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m'gs_fl'\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfind_all\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'a'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msplit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m-\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0;31m#     cited_section = atl_obj.find('div', {'class':'gs_ri'}).find('div', {'class':'gs_fl'}).find_all('a')[2]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m \u001b[0;31m#     total_citation = int(cited_section.text.split()[-1])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'NoneType' object has no attribute 'find'"
     ]
    }
   ],
   "source": [
    "all_info = []\n",
    "for itm in all_volumes:\n",
    "    print('year', itm[0])\n",
    "    if itm[0]==1994:\n",
    "        continue\n",
    "    cnt=get_volume_info(itm[-1])\n",
    "    all_info.append([itm[0], cnt])\n",
    "    with open('%d.pkl'%itm[0], 'wb') as filout:\n",
    "        pk.dump(cnt, filout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}