helper_code_data_allocation_manual.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1402966e",
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import pickle\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from glob import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "11547854",
   "metadata": {},
   "outputs": [],
   "source": [
    "mscinet_pages = glob(\"../mathscinet/publication_pages/*.html\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "1ca8e389",
   "metadata": {},
   "outputs": [],
   "source": [
    "mscinet_ids = [page.rsplit(\"/\")[-1].split(\"_\")[-1].replace(\".html\",\"\") for page in mscinet_pages]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "82c2cb7b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#int(mscinet_pages[0].rsplit(\"/\")[-1].split(\"_\")[-1].replace(\".html\",\"\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "fca05bb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "data_folder = \"../data/mgp_data/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fc3a5507",
   "metadata": {},
   "outputs": [],
   "source": [
    "mgp_data = pd.read_csv(data_folder+\"mgpnodeList_with_year_completion_updated_with_mod_dblp_link.csv\",lineterminator='\\n', low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "051a891a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['Id', 'Name', 'Year', 'University', 'Country', 'Title', 'MSC', 'BIO',\n",
       "       'MSN', 'Dblp_Link'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mgp_data.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "1e66fff1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#[mgp_data~(mgp_data['MSN'].isna())& (mgp_data['Id']==153165)][\"MSN\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "288e1dcb",
   "metadata": {},
   "outputs": [],
   "source": [
    "mgp_mscinet =mgp_data[~(mgp_data['MSN'].isna())][['Name','MSN']].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "16d9f095",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Name</th>\n",
       "      <th>MSN</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>148112</th>\n",
       "      <td>Juan Carlos  Gutiérrez</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/358553</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>119996</th>\n",
       "      <td>Mario  Helm</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/679625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15611</th>\n",
       "      <td>Robert Spencer David Thomas</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/197523</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>218852</th>\n",
       "      <td>Eric Prager Astor</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/1125023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>35127</th>\n",
       "      <td>Alfredo Rosário Jones</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/95365</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>21439</th>\n",
       "      <td>Herbert  Leinfelder</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/190066</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>197673</th>\n",
       "      <td>Yu  Wang</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/894268</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147764</th>\n",
       "      <td>Liqun  Fang</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/896800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>219099</th>\n",
       "      <td>Saleh  Soltan</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/1221075</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>149126</th>\n",
       "      <td>Peter  Steinacker</td>\n",
       "      <td>http://www.ams.org/mathscinet/MRAuthorID/166855</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                               Name  \\\n",
       "148112       Juan Carlos  Gutiérrez   \n",
       "119996                  Mario  Helm   \n",
       "15611   Robert Spencer David Thomas   \n",
       "218852            Eric Prager Astor   \n",
       "35127         Alfredo Rosário Jones   \n",
       "21439           Herbert  Leinfelder   \n",
       "197673                     Yu  Wang   \n",
       "147764                  Liqun  Fang   \n",
       "219099                Saleh  Soltan   \n",
       "149126            Peter  Steinacker   \n",
       "\n",
       "                                                     MSN  \n",
       "148112   http://www.ams.org/mathscinet/MRAuthorID/358553  \n",
       "119996   http://www.ams.org/mathscinet/MRAuthorID/679625  \n",
       "15611    http://www.ams.org/mathscinet/MRAuthorID/197523  \n",
       "218852  http://www.ams.org/mathscinet/MRAuthorID/1125023  \n",
       "35127     http://www.ams.org/mathscinet/MRAuthorID/95365  \n",
       "21439    http://www.ams.org/mathscinet/MRAuthorID/190066  \n",
       "197673   http://www.ams.org/mathscinet/MRAuthorID/894268  \n",
       "147764   http://www.ams.org/mathscinet/MRAuthorID/896800  \n",
       "219099  http://www.ams.org/mathscinet/MRAuthorID/1221075  \n",
       "149126   http://www.ams.org/mathscinet/MRAuthorID/166855  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mgp_mscinet.sample(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "f4a619be",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mgp_mscinet.to_csv(data_folder+\"mathscinet_link_for_mgp_researchers.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "f91052a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet = pd.read_csv(data_folder+\"info_mathscinet.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "eb965703",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mathscinet[\"publ_by_area\"].sample(5).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "807a7e8b",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mathscinet[~(mathscinet[\"publ_url\"].isna())][['mgp_id','publ_url']].sample(10).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "f7904b59",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet[\"avl\"] = mathscinet[\"mscinet_id\"].isin(mscinet_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "9955eb09",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet_mod = mathscinet[mathscinet[\"avl\"]==False].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "55fe0018",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mathscinet.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "04da08f2",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mathscinet_mod[[\"mgp_id\",\"mscinet_id\"]].to_csv(data_folder+\"mgpId_mscinetId_not_avl.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "001d171f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[188043, '735043',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=735043&yrop=eq&dr=pubyear&arg3=2002',\n",
       "        './publications.html?pg1=INDI&s1=735043'],\n",
       "       [49952, '249540',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=249540&yrop=eq&dr=pubyear&arg3=1985',\n",
       "        './publications.html?pg1=INDI&s1=249540'],\n",
       "       [265024, '727550',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=727550&yrop=eq&dr=pubyear&arg3=2003',\n",
       "        './publications.html?pg1=INDI&s1=727550'],\n",
       "       [172945, '110235',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=110235&yrop=eq&dr=pubyear&arg3=1965',\n",
       "        './publications.html?pg1=INDI&s1=110235,./publications.html?pg1=MIID&s1=110235'],\n",
       "       [184461, '864243',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=864243&yrop=eq&dr=pubyear&arg3=2009',\n",
       "        './publications.html?pg1=INDI&s1=864243'],\n",
       "       [238383, '1197796',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=1197796&yrop=eq&dr=pubyear&arg3=2017',\n",
       "        './publications.html?pg1=INDI&s1=1197796'],\n",
       "       [38124, '740410',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=740410&yrop=eq&dr=pubyear&arg3=1999',\n",
       "        './publications.html?pg1=INDI&s1=740410'],\n",
       "       [125335, '534773',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=534773&yrop=eq&dr=pubyear&arg3=1981',\n",
       "        './publications.html?pg1=INDI&s1=534773'],\n",
       "       [89739, '290264',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=290264&yrop=eq&dr=pubyear&arg3=1990',\n",
       "        './publications.html?pg1=INDI&s1=290264'],\n",
       "       [92446, '634671',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=634671&yrop=eq&dr=pubyear&arg3=1998',\n",
       "        './publications.html?pg1=INDI&s1=634671,./publications.html?pg1=MIID&s1=634671'],\n",
       "       [138878, '507428',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=507428&yrop=eq&dr=pubyear&arg3=1977',\n",
       "        './publications.html?pg1=INDI&s1=507428'],\n",
       "       [229991, '715920',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=715920&yrop=eq&dr=pubyear&arg3=2003',\n",
       "        './publications.html?pg1=INDI&s1=715920'],\n",
       "       [163790, '1185461',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=1185461&yrop=eq&dr=pubyear&arg3=2016',\n",
       "        './publications.html?pg1=INDI&s1=1185461'],\n",
       "       [130011, '924245',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=924245&yrop=eq&dr=pubyear&arg3=2008',\n",
       "        './publications.html?pg1=INDI&s1=924245'],\n",
       "       [245490, '776120',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=776120&yrop=eq&dr=pubyear&arg3=2005',\n",
       "        './publications.html?pg1=INDI&s1=776120'],\n",
       "       [17202, '552447',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=552447&yrop=eq&dr=pubyear&arg3=1971',\n",
       "        './publications.html?pg1=INDI&s1=552447'],\n",
       "       [104926, '309653',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=309653&yrop=eq&dr=pubyear&arg3=1991',\n",
       "        './publications.html?pg1=INDI&s1=309653'],\n",
       "       [217774, '1001089',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=1001089&yrop=eq&dr=pubyear&arg3=2013',\n",
       "        './publications.html?pg1=INDI&s1=1001089'],\n",
       "       [70830, '667028',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=667028&yrop=eq&dr=pubyear&arg3=1992',\n",
       "        './publications.html?pg1=INDI&s1=667028'],\n",
       "       [128412, '665018',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=665018&yrop=eq&dr=pubyear&arg3=2000',\n",
       "        './publications.html?pg1=INDI&s1=665018'],\n",
       "       [126525, '145550',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=145550&yrop=eq&dr=pubyear&arg3=1968',\n",
       "        './publications.html?pg1=INDI&s1=145550'],\n",
       "       [38371, '643268',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=643268&yrop=eq&dr=pubyear&arg3=1998',\n",
       "        './publications.html?pg1=INDI&s1=643268'],\n",
       "       [158547, '811937',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=811937&yrop=eq&dr=pubyear&arg3=2006',\n",
       "        './publications.html?pg1=INDI&s1=811937'],\n",
       "       [115636, '704176',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=704176&yrop=eq&dr=pubyear&arg3=2002',\n",
       "        './publications.html?pg1=INDI&s1=704176'],\n",
       "       [219503, '1123120',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=1123120&yrop=eq&dr=pubyear&arg3=2015',\n",
       "        './publications.html?pg1=INDI&s1=1123120'],\n",
       "       [35237, 'None', nan, nan],\n",
       "       [103943, '252280',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=252280&yrop=eq&dr=pubyear&arg3=1988',\n",
       "        './publications.html?pg1=INDI&s1=252280'],\n",
       "       [24672, '234187',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=234187&yrop=eq&dr=pubyear&arg3=1986',\n",
       "        './publications.html?pg1=INDI&s1=234187'],\n",
       "       [271318, '904932',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=904932&yrop=eq&dr=pubyear&arg3=2010',\n",
       "        './publications.html?pg1=INDI&s1=904932'],\n",
       "       [131518, '562217',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=562217&yrop=eq&dr=pubyear&arg3=1949',\n",
       "        './publications.html?pg1=INDI&s1=562217'],\n",
       "       [219843, '191032',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=191032&yrop=eq&dr=pubyear&arg3=1983',\n",
       "        './publications.html?pg1=INDI&s1=191032'],\n",
       "       [92404, '834137',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=834137&yrop=eq&dr=pubyear&arg3=2005',\n",
       "        './publications.html?pg1=INDI&s1=834137'],\n",
       "       [70706, '315740',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=315740&yrop=eq&dr=pubyear&arg3=1990',\n",
       "        './publications.html?pg1=INDI&s1=315740'],\n",
       "       [2503, '603133',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=603133&yrop=eq&dr=pubyear&arg3=1993',\n",
       "        './publications.html?pg1=INDI&s1=603133'],\n",
       "       [117718, '197348',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=197348&yrop=eq&dr=pubyear&arg3=1963',\n",
       "        './publications.html?pg1=INDI&s1=197348,./publications.html?pg1=MIID&s1=197348'],\n",
       "       [130316, '145410',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=145410&yrop=eq&dr=pubyear&arg3=1960',\n",
       "        './publications.html?pg1=INDI&s1=145410,./publications.html?pg1=MIID&s1=145410'],\n",
       "       [115586, '412263',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=412263&yrop=eq&dr=pubyear&arg3=1951',\n",
       "        './publications.html?pg1=INDI&s1=412263'],\n",
       "       [30034, '190522',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=190522&yrop=eq&dr=pubyear&arg3=1940',\n",
       "        './publications.html?pg1=INDI&s1=190522,./publications.html?pg1=MIID&s1=190522'],\n",
       "       [79772, '674802',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=674802&yrop=eq&dr=pubyear&arg3=2001',\n",
       "        './publications.html?pg1=INDI&s1=674802,./publications.html?pg1=MIID&s1=674802'],\n",
       "       [142338, '221843',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=221843&yrop=eq&dr=pubyear&arg3=1982',\n",
       "        './publications.html?pg1=INDI&s1=221843'],\n",
       "       [118737, '897117',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=897117&yrop=eq&dr=pubyear&arg3=2010',\n",
       "        './publications.html?pg1=INDI&s1=897117'],\n",
       "       [262004, '1320922',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=1320922&yrop=eq&dr=pubyear&arg3=2019',\n",
       "        './publications.html?pg1=INDI&s1=1320922'],\n",
       "       [187898, '989987',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=989987&yrop=eq&dr=pubyear&arg3=2011',\n",
       "        './publications.html?pg1=INDI&s1=989987'],\n",
       "       [144576, '868423',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=868423&yrop=eq&dr=pubyear&arg3=2009',\n",
       "        './publications.html?pg1=INDI&s1=868423'],\n",
       "       [36863, '281431',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=281431&yrop=eq&dr=pubyear&arg3=1973',\n",
       "        './publications.html?pg1=INDI&s1=281431'],\n",
       "       [101791, '158120',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=158120&yrop=eq&dr=pubyear&arg3=1986',\n",
       "        './publications.html?pg1=INDI&s1=158120'],\n",
       "       [43736, '431596',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=431596&yrop=eq&dr=pubyear&arg3=1980',\n",
       "        './publications.html?pg1=INDI&s1=431596'],\n",
       "       [214680, '1218693',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=1218693&yrop=eq&dr=pubyear&arg3=2017',\n",
       "        './publications.html?pg1=INDI&s1=1218693'],\n",
       "       [180076, '954575',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=954575&yrop=eq&dr=pubyear&arg3=2011',\n",
       "        './publications.html?pg1=INDI&s1=954575'],\n",
       "       [144401, '178930',\n",
       "        '/mathscinet/search/publdoc.html?pg1=INDI&s1=178930&yrop=eq&dr=pubyear&arg3=1975',\n",
       "        './publications.html?pg1=INDI&s1=178930,./publications.html?pg1=MIID&s1=178930']],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mathscinet_mod.iloc[1950:2000][[\"mgp_id\",\"mscinet_id\",\"eip_link\",\"publ_url\"]].values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "32403f2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mathscinet_mod.iloc[1950:2000][[\"mgp_id\",\"mscinet_id\",\"eip_link\",\"publ_url\"]].values "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "b9a5c108",
   "metadata": {},
   "outputs": [],
   "source": [
    "scrape_data = mathscinet_mod.iloc[2000:].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "b6d1e676",
   "metadata": {},
   "outputs": [],
   "source": [
    "scrape_data[[\"publ_url_only\",\"related_publ\"]] = scrape_data[\"publ_url\"].str.split(',', 1, expand=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "id": "d73075e9",
   "metadata": {},
   "outputs": [],
   "source": [
    "scrape_data.loc[scrape_data[\"publ_url_only\"].isna(),\"publ_url_only\"] = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "id": "40af982b",
   "metadata": {},
   "outputs": [],
   "source": [
    "scrape_data[\"full_publ_url\"]= scrape_data[\"publ_url_only\"].apply(lambda x : \"https://mathscinet.ams.org/mathscinet/search\"+x.lstrip(\".\") if x!=\"\" else x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "id": "f03bcbae",
   "metadata": {},
   "outputs": [],
   "source": [
    "extract_publ_data = scrape_data[~(scrape_data[\"full_publ_url\"]==\"\")].copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "id": "a90554a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(141342, 17)"
      ]
     },
     "execution_count": 92,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "extract_publ_data.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "id": "3febbe5f",
   "metadata": {},
   "outputs": [],
   "source": [
    "file1 = extract_publ_data.iloc[0:30000][\"full_publ_url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "05f43d77",
   "metadata": {},
   "outputs": [],
   "source": [
    "file2 = extract_publ_data.iloc[30000:60000][\"full_publ_url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "id": "ac9e6c96",
   "metadata": {},
   "outputs": [],
   "source": [
    "file3 = extract_publ_data.iloc[60000:90000][\"full_publ_url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "id": "8d8af553",
   "metadata": {},
   "outputs": [],
   "source": [
    "file4 = extract_publ_data.iloc[90000:120000][\"full_publ_url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "470f444c",
   "metadata": {},
   "outputs": [],
   "source": [
    "file5 = extract_publ_data.iloc[120000:][\"full_publ_url\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "id": "00f04ec9",
   "metadata": {},
   "outputs": [],
   "source": [
    "file1.to_csv(\"~/Desktop/Publ_files/Jayanta_Das.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f99eb9a3",
   "metadata": {},
   "outputs": [],
   "source": [
    "file2.to_csv(\"~/Desktop/Publ_files/Ananya_Guha.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "75a9bb8c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['https://mathscinet.ams.org/mathscinet/search/publications.html?pg1=INDI&s1=734909',\n",
       "       'https://mathscinet.ams.org/mathscinet/search/publications.html?pg1=INDI&s1=294437',\n",
       "       'https://mathscinet.ams.org/mathscinet/search/publications.html?pg1=INDI&s1=780137',\n",
       "       'https://mathscinet.ams.org/mathscinet/search/publications.html?pg1=INDI&s1=527458',\n",
       "       'https://mathscinet.ams.org/mathscinet/search/publications.html?pg1=INDI&s1=332806'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 57,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "scrape_data[\"full_publ_url\"].sample(5).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "39d165a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet[\"co_author_names\"] = mathscinet[\"co_author_list\"].apply(lambda x : [author[-1] for author in eval(x)] if not pd.isna(x) else \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "5a976143",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mathscinet[[\"co_author_list\",\"co_author_names\"]].sample().values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ff249b6b",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet[\"published_as1_1\"]=mathscinet[\"published_as1\"].apply(lambda x : x.strip().replace(\"Published as:\",\"\") if not pd.isna(x) else \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "b3c4ef97",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet[\"published_as1_1\"]=mathscinet[\"published_as1_1\"].apply(lambda x : re.sub(r'\\(\\d*\\)',\";\",x))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "fccbeb91",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet['published_as2_1'] = mathscinet['published_as2'].apply(lambda x : re.sub(r'\\(\\d*\\)',\";\",x) if not pd.isna(x) else \"\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "699ffd42",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet['published_as'] = mathscinet['published_as1_1']+\";\"+mathscinet['published_as2_1']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "984e0822",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet['published_as'] = mathscinet['published_as'].apply(lambda x : \";\".join([name.strip() for name in x.split(\";\") if name != \"\"]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "26a6ec78",
   "metadata": {},
   "outputs": [],
   "source": [
    "mathscinet.to_csv(data_folder+\"mod_info_mathscinet.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ba4e0964",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[' De Munter, P. ;', 'De Munter, Paul ;,de Munter, P. ;',\n",
       "        'De Munter, P.;De Munter, Paul;,de Munter, P.'],\n",
       "       [' García-Beltrán, D. ;', 'García-Beltrán, Dennise ;',\n",
       "        'García-Beltrán, D.;García-Beltrán, Dennise'],\n",
       "       [' Strait, Peggy ;', 'Strait, Peggy Tang ;',\n",
       "        'Strait, Peggy;Strait, Peggy Tang'],\n",
       "       ['', '', ''],\n",
       "       [' Vermeer, P. J. ;', 'Vermeer, Pamela J. ;',\n",
       "        'Vermeer, P. J.;Vermeer, Pamela J.'],\n",
       "       ['', '', ''],\n",
       "       [' Wong, R. ;',\n",
       "        'Wong, Roderick ;,Wong, Roderick S. ;,Wong, Roderick S. C. ;,Wong, Roderick Sue-Chuen ;',\n",
       "        'Wong, R.;Wong, Roderick;,Wong, Roderick S.;,Wong, Roderick S. C.;,Wong, Roderick Sue-Chuen'],\n",
       "       ['', '', ''],\n",
       "       [' Nasser, G. ;', 'Nasser, George ;,Nasser, Mahmoud George ;',\n",
       "        'Nasser, G.;Nasser, George;,Nasser, Mahmoud George'],\n",
       "       ['', '', '']], dtype=object)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mathscinet[['published_as1_1','published_as2_1','published_as']].sample(10).values"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "b506835e",
   "metadata": {
    "collapsed": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['mgp_id', 'mscinet_id', 'eip', 'eip_link', 'total_publ_count',\n",
       "       'total_related_publ_count', 'total_citation_count', 'publ_url',\n",
       "       'co_author_list', 'publ_by_area', 'publ_by_citation', 'published_as1',\n",
       "       'published_as2', 'co_author_names', 'published_as1_1',\n",
       "       'published_as2_1', 'published_as'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mathscinet.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "f6f025f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "mgp_id2coauthor = zip(mathscinet['mgp_id'],mathscinet['co_author_names'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "cbada01d",
   "metadata": {},
   "outputs": [],
   "source": [
    "mgp_id2name_variation = zip(mathscinet['mgp_id'],mathscinet['published_as'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "75dab7ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(data_folder+'mgp_id2coauthor.pickle', 'wb') as handle:\n",
    "    pickle.dump(mgp_id2coauthor, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "f34199db",
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(data_folder+'mgp_id2name_variation.pickle', 'wb') as handle:\n",
    "    pickle.dump(mgp_id2coauthor, handle, protocol=pickle.HIGHEST_PROTOCOL)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "38f238c1",
   "metadata": {},
   "outputs": [],
   "source": [
    "#load pickle\n",
    "# with open('filename.pickle', 'rb') as handle:\n",
    "#     b = pickle.load(handle)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "c4641cc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kwargs = dict(histtype='stepfilled', alpha=0.3, normed=True, bins=40)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "2f6c60c3",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kwargs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "844c86e7",
   "metadata": {},
   "outputs": [],
   "source": [
    "#kwargs['bins']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "feac4dcb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(143741, 14)"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mathscinet_mod.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "114fe890",
   "metadata": {},
   "outputs": [],
   "source": [
    "#mathscinet_mod.iloc[143700:500000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f8f58c58",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}