pageRank.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "import igraph as ig\n",
    "import pickle\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from functools import reduce\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "font = {'family' : 'normal',\n",
    "        'weight' : 'bold',\n",
    "        'size'   : 18}\n",
    "\n",
    "plt.rcParams.update({'font.size': 18})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PageRank for tracking innovative compounds\n",
    "\n",
    "Goal => use pagerank measures over time to find innovative cpds"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Goal 1: Innovation through distributions\n",
    "\n",
    "Test actual pagerank values\n",
    "\n",
    "- Increasing values = higher influence\n",
    "- Persistently high value after increase = innovation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "## Testing for values\n",
    "fp = \"G:Shared drives/SureChemBL_Patents/PageRank/\"\n",
    "pr_1980_01 = pickle.load(file=open(fp + \"pagerank_1980-01.p\", \"rb\"))\n",
    "pr_1980_02 = pickle.load(file=open(fp + \"pagerank_1980-02.p\", \"rb\"))\n",
    "pr_2019_11 = pickle.load(file=open(fp + \"pagerank_2019-11.p\", \"rb\"))\n",
    "pr_2019_12 = pickle.load(file=open(fp + \"pagerank_2019-12.p\", \"rb\"))\n",
    "\n",
    "df_1980_01 = pd.DataFrame(pr_1980_01, columns=[\"ID\", \"PR_1980_01\"])\n",
    "df_1980_02 = pd.DataFrame(pr_1980_02, columns=[\"ID\", \"PR_1980_02\"])\n",
    "df_2019_11 = pd.DataFrame(pr_2019_11, columns=[\"ID\", \"PR_2019_11\"])\n",
    "df_2019_12 = pd.DataFrame(pr_2019_12, columns=[\"ID\", \"PR_2019_12\"])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0, 0.5, 'Count')"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAawAAAEWCAYAAAA6maO/AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjUuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/NK7nSAAAACXBIWXMAAAsTAAALEwEAmpwYAAAx2ElEQVR4nO3deXyU1d3//9eHXZDNhlsFCUHAIhRMCyjYYqLiBiqWVnGBoj+BFjdAHy64VKSoiKJsLgHqz7pVVLgrlbhUVvWWm1UEZVMJCOiNAWULCQjn+8d1ZZyZTJKZZMJkkvfz8ZjHZM51rnOd6zDkk+tc5zrHnHOIiIhUdjUSXQEREZFoKGCJiEhSUMASEZGkoIAlIiJJQQFLRESSQq1EV6CySUlJcWlpaYmuhohIUlmxYkWuc65ZRR6jygYsM3sBuBY4FJT8R+fcuyXtl5aWxvLlyyuyaiIiVY6ZbanoY1TZgOWb5py7JdGVEBGR8tM9LBERSQoJD1hmNsrM3jCzr83MmVlOCXlrmNlIM1tvZvlm9o2ZTTCzBsXscp2Z7TazdWZ2n5lV9StKEZEqK+EBC3gEOA/4CvihlLxPAU8CXwC3Am8AtwH/NrPwc5kM/BJIAQYC1wMPxq3WIiJyTFWGK442zrmvAcxsLXB8pExm1hEvSM12zv0hKH0zXnC6Gni1MN05tzJo9+Vm9iDwEPBA3M9AREQqXMKvsAqDVRSuAQyYGJY+HcgDBpSy/1F/fxERSUIJD1gx6IYXdJYGJzrn8oFP/e0BZtbfzBqbpzNed+Abx6iuIiISZ5WhSzBazYFc51xBhG3bgbPNrI5zrvC5q5uA54DawLfAS8CjkQo2s6HAUIDU1NR411ukUiooKGD37t3s27ePI0eOJLo6UsnUrFmThg0bcsIJJ1C3bt1EVwdIroBVH4gUrADyg/IcAnDOZURbsHNuGjANoGvXrmVeIKzTPzqVab81g9aU9ZAiZVJQUMDWrVtp2rQpaWlp1K5dGzP1mIvHOcfhw4fZu3cvW7duJTU1tVIErWTqEswDimuxekF5RKQUu3fvpmnTpqSkpFCnTh0FKwlhZtSpU4eUlBSaNm3K7t27E10lILkC1g4gxcwiBa0WeN2FhyJsE5Ew+/bto1GjRomuhiSBRo0asW/fvkRXA0iugLUMr75nBieaWT0gHdAEgCJROnLkCLVr1050NSQJ1K5du9Lc40ymgDUTcMCIsPQhePeuXjnWFRJJZuoGlGhUpu9JwgddmNlAoJX/sRlQx8zu9z9vcc69BOCcW2NmTwO3mNlsIBs4HW+mi0UEPTQsIiJVT8IDFnAjED6i72/++yK84eiFRgA5eEPQ+wC5wBTgr865oxVaSxERSaiEByznXGYMeY8AE/yXiIhUIwkPWCJSuaTdMzfRVQiRM65Puct49NFHWblyJStWrGDz5s20atWKnJyciHmdc2RlZZGVlcX69eupW7cu3bt3Z/To0XTv3r1I/tzcXMaPH8+cOXPYunUrjRo1okOHDgwfPpy+ffsWyZ+dnc3YsWNZvXo1devW5fzzz2f8+PG0bt066vM5evQokyZNIisri5ycHJo1a8ZVV13FmDFjaNAgdPGK119/nXfeeYeVK1fyxRdf8NNPP7F582aScWX1ZBp0ISJSJvfeey/z58+nTZs2NG3atMS8N910E8OGDaNJkyaMHz+eUaNGsWnTJjIyMli4cGFI3ry8PM4++2ymTJnChRdeyOTJk7n99tv57rvvuOKKK3j22WdD8s+ePZtLL72UgwcP8vjjj3PnnXeyePFifvvb37Jjx46oz2fkyJHcfvvtdOjQgSlTpnDllVcyefJkLrvsMo4eDb078swzz/Daa69x3HHH0aZNm6iPURnpCktEqryvvvqKU089FYBf/epX7N+/P2K+Tz/9lOeee46LL76Y7OzswAi5P//5z7Rv356hQ4eyfv16atTw/tZ/66232LRpExMnTmT48OGBcoYOHcopp5xCVlYWw4YNA+Dw4cPceuuttGzZkg8//JDjj/cWprjkkkvo0qULo0ePZtq0aaWey+eff86UKVPo168fs2bNCqS3bt2a2267jddee41rr702kP7iiy/SvHlzatWqxS233MKGDRtiabpKRVdYIlLlFQar0ixYsACAQYMGhQznbtKkCX379mXTpk18/PHHgfS9e/cC0Lx585ByGjduTIMGDUK65xYtWsSOHTsYPHhwIFgBpKenk5mZycyZMzl8+HCpdfznP/+Jc44RI0aEpA8ZMoT69evz8ssvh6SnpqZSq1bVuDZRwBIR8RUUeNOV1q9fv8i2wrQlS5YE0s477zxq1arFqFGjyM7OZtu2baxZs4Ybb7yRH3/8kfvuuy+Qd9myZQD06NGjSNndu3dn7969bNy4sdQ6Llu2jBo1anDmmSFzKFCvXj3S09MDx6mKFLBERHwdO3YEYP78+SHpzjkWLVoEwDfffBNIb9euHTNnzqSgoIA+ffrQsmVLOnfuTHZ2NvPnz6d3796BvIX3qFq0aFHkuIVp27dvL7WOO3bsICUlJeJktC1atCA3N5dDh6rmLHVV4zpRRCQOLrnkEjp06MAzzzxD8+bN6devH3l5eTz55JOsXbsW8AZaBGvSpAmdO3dm8ODBpKens337diZMmEDfvn2ZN28eZ5xxRsh+kQJNvXr1IpYdSV5eXrEzpweXU6dOnSjPOnkoYImI+GrVqsU777zDoEGDuPvuu7n77rsB6Ny5M+PGjeOOO+4ImTT4vffeo3fv3sydO5eLL744kN6vXz/at2/PzTffzEcffQT83KVY2O0YLD8/PyTPnj17OHjwYEieZs2aUbNmTerXr8/OnTsj1j+8nKpGXYIiIkFSU1NZsGABW7ZsYdGiRaxdu5bVq1cHrl7at28fyPvYY4/RoEGDkGAFcNJJJ9GzZ0+WLFkS6J4rHJgRqduvMK2wa3D48OGcfPLJIa/CrsjmzZuTm5sbMfBt3749sGRMVaQrLBGRCFJTU0NWIM/OzqZGjRpcdNFFgbTt27dz9OhRnHNFJon96aefOHLkSOC5qG7dugHwySef0KtXr5C8S5YsoVGjRpx22mkA3HXXXQwYMCAkz0knnRQo5/3332fp0qX07NkzsD0/P59PP/2Uc845p7ynXmnpCktEpBRz5sxh7ty5DBw4kFatWgXSO3TowIEDB3jjjTdC8m/evJnFixfTqVOnwJVZRkYGJ598MjNmzAh5Dmz16tUsXLiQK6+8MrDkS4cOHejVq1fIq7Cc/v37Y2ZMnDgx5JjTp08nLy+P6667riKaoFLQFZaIVHkvvfQSW7ZsAeD777/n0KFDjB07FoBWrVoxcODAQN4bb7wR5xzp6ekcd9xxfPTRR7zyyit069aNSZMmhZR777338u677zJgwAAWLlxIeno627Zt49lnnyU/P59HHnkkkLd27dpMmjSJ/v3707NnT4YMGcLevXt56qmnaNasGQ899FBU59KpUyduvvlmpk6dSr9+/ejduzfr1q1j8uTJZGRkhDw0DLB48WIWL14MwPLl3rKBU6dOpUmTJgDcf//9JAtzziW6DpVK165dXeE/aqw6/aNTmfZbM2hNmfYTKat169Zx+umnR9xWFecSzMzMDAxLDxc+5VJWVhbTpk3jyy+/5NChQ7Rt25ZrrrmGkSNHctxxxxXZf9WqVTz88MN8+OGH7Nq1i4YNG3LWWWdxzz33kJmZWST/22+/zdixY/nss88Ccwk+9thjMU2bdOTIESZOnMi0adPIyckhJSWF/v37M2bMmJCHkgFGjx5dYjCMJgaU9H0pZGYrnHNdozuDslHACqOAJdVBNL+ARApVloClLsFKIB5/0cbjr1ARkcpMAasSyKl3bemZSpCWr8WWRaTq0yhBERFJCgpYIiKSFBSwREQkKShgiYhIUlDAEhGRpKCAJSIiSUEBS0REkoICloiIJAUFLBERSQoKWCIikhQUsEREJCloLkERCTW6caJrEGr0nnIX8eijj7Jy5UpWrFjB5s2badWqFTk5ORHzOufIysoiKyuL9evXU7duXbp3787o0aPp3r17kfy5ubmMHz+eOXPmsHXrVho1akSHDh0YPnw4ffv2LZI/OzubsWPHsnr16sDyIuPHj6d169ZRn8/Ro0eZNGkSWVlZ5OTk0KxZM6666irGjBlDgwYNAvl++OEHXnzxRebOncu6devIzc0lNTWVjIwMHnjgAVq2bBn1MSsDXWGJSJV37733Mn/+fNq0aUPTpk1LzHvTTTcxbNgwmjRpwvjx4xk1ahSbNm0qsm4WQF5eHmeffTZTpkzhwgsvZPLkydx+++189913XHHFFTz77LMh+WfPns2ll17KwYMHefzxx7nzzjtZvHgxv/3tb9mxY0fU5zNy5Ehuv/12OnTowJQpU7jyyiuZPHkyl112GUePHg3k+9///V/uuOMOzIxbbrmFqVOn0rt3b15++WU6derEF198EfUxKwNdYYlIlffVV19x6qmnAvCrX/0qZIn6YJ9++inPPfccF198MdnZ2ZgZAH/+859p3749Q4cOZf369dSo4f2t/9Zbb7Fp0yYmTpzI8OHDA+UMHTqUU045haysLIYNGwbA4cOHufXWW2nZsiUffvhhYKHFSy65hC5dujB69GimTZtW6rl8/vnnTJkyhX79+jFr1qxAeuvWrbntttt47bXXAqsOt2/fng0bNhRZHLJPnz5ccMEF/PWvf+XNN9+Mqg0rA11hiUiVVxisSrNgwQIABg0aFAhWAE2aNKFv375s2rSJjz/+OJC+d+9eAJo3bx5STuPGjWnQoEFI99yiRYvYsWMHgwcPDlkVOD09nczMTGbOnMnhw4dLreM///lPnHOMGDEiJH3IkCHUr1+fl19+OZCWlpYWcSXjXr16ccIJJ7B27dpSj1eZKGCJiPgKCgoAqF+/fpFthWlLliwJpJ133nnUqlWLUaNGkZ2dzbZt21izZg033ngjP/74I/fdd18g77JlywDo0aNHkbK7d+/O3r172bhxY6l1XLZsGTVq1ODMM88MSa9Xrx7p6emB45Rkz5497Nu3jxNPPLHUvJWJApaIiK9jx44AzJ8/PyTdOceiRYsA+OabbwLp7dq1Y+bMmRQUFNCnTx9atmxJ586dyc7OZv78+fTu3TuQt/AeVYsWLYoctzBt+/btpdZxx44dpKSkULdu3Yjl5ObmcujQoRLLePjhhzl8+DCDBg0q9XiVie5hiYj4LrnkEjp06MAzzzxD8+bN6devH3l5eTz55JOB7rO8vLyQfZo0aULnzp0ZPHgw6enpbN++nQkTJtC3b1/mzZvHGWecEbJfpEBTr169iGVHkpeXF7GM8HLq1KkTMc+bb77JE088wcUXX8wNN9xQ6vEqEwUsERFfrVq1eOeddxg0aBB33303d999NwCdO3dm3Lhx3HHHHTRq1CiQ/7333qN3797MnTuXiy++OJDer18/2rdvz80338xHH30E/NylWNjtGCw/Pz8kz549ezh48GBInmbNmlGzZk3q16/Pzp07I9Y/vJxw2dnZXHfddXTp0oWZM2eG3KdLBuoSFBEJkpqayoIFC9iyZQuLFi1i7dq1rF69OnD10r59+0Dexx57jAYNGoQEK4CTTjqJnj17smTJkkD3XOHAjEjdfoVphV2Dw4cP5+STTw55FXZFNm/enNzc3IiBb/v27aSkpES8unr33Xfp168fHTt25P333w8JvMlCV1giIhGkpqaSmpoa+JydnU2NGjW46KKLAmnbt2/n6NGjOOeKXK389NNPHDlyJPBcVLdu3QD45JNP6NWrV0jeJUuW0KhRI0477TQA7rrrLgYMGBCS56STTgqU8/7777N06VJ69uwZ2J6fn8+nn37KOeecU+Rc3n33Xa644grat2/PBx98UOqzaJWVrrBEREoxZ84c5s6dy8CBA2nVqlUgvUOHDhw4cIA33ngjJP/mzZtZvHgxnTp1ClyZZWRkcPLJJzNjxoyQ58BWr17NwoULufLKK6ldu3ag3F69eoW8Csvp378/ZsbEiRNDjjl9+nTy8vK47rrrQtLff/99fv/73/PLX/6SefPmccIJJ8StXY41XWGJSJX30ksvsWXLFgC+//57Dh06xNixYwFo1aoVAwcODOS98cYbcc6Rnp7Occcdx0cffcQrr7xCt27dmDRpUki59957L++++y4DBgxg4cKFpKens23bNp599lny8/N55JFHAnlr167NpEmT6N+/Pz179mTIkCHs3buXp556imbNmvHQQw9FdS6dOnXi5ptvZurUqfTr14/evXuzbt06Jk+eTEZGRuChYYDly5fTt29fnHPccMMNvPPOO0XKC7+Sq8zMOZfoOlQqXbt2dcuXLy/Tvp3+0alM+63ZvLVM+xVKy3+VnHF9ylWGVC/r1q3j9NNPj7yxCs4lmJmZGRiWHi58yqWsrCymTZvGl19+yaFDh2jbti3XXHMNI0eO5Ljjjiuy/6pVq3j44Yf58MMP2bVrFw0bNuSss87innvuITMzs0j+t99+m7Fjx/LZZ58F5hJ87LHHIj7gW5wjR44wceJEpk2bRk5ODikpKfTv358xY8aEPJT8wgsvlDoSMJoYUOL3xWdmK5xzXaM7g7JRwAqjgCXVQTS/gEQKVZaApXtYIiKSFBSwREQkKShgiYhIUlDAEhGRpKCAJSIiSUEBS0REkoICloiIJAUFLBERSQoKWCIikhQUsEREJCkoYImISFJQwBIRkaSggCUiIklB62GJSIiyrjpQUdYMWlOu/Tdu3MjLL7/M+++/z1dffUV+fj5t2rThyiuvZMSIETRo0CAk/4YNG7j77rtZtGgRhw4d4je/+Q0PPfQQ5513Xki+b7/9lqlTp7JixQpWrFhBbm4ugwYN4oUXXohYj/379zNmzBhmzZrFtm3baNq0KZdccgljx46lRYsWUZ9PVlYWixcvZsWKFWzatCmw4nE45xyvvPIKb7/9NsuXL2fHjh2kpKSQnp7Offfdx1lnnRX1MSsLBSwRqdKef/55nn76aS6//HKuu+46ateuzYIFC7j//vt5/fXXWbJkSWCdq6+++oqzzz6bWrVqcdddd9G4cWOmT5/ORRddxDvvvBOytP2GDRt45JFHaNmyJd26dYu4OGKhgwcPkpGRwapVq/jTn/5Ejx492Lx5M08//TTz5s1j6dKlnHTSSVGdz6OPPsquXbv49a9/zYEDB9i2bVvEfAUFBQwcOJD09HSuvvpqWrduzbfffstzzz1Hjx49ePHFF5Nq8UZQwBKRKu6Pf/wjo0aNonHjnxem/Mtf/kK7du14+OGH+fvf/84tt9wCwKhRo/jxxx9ZsWIF6enpAPzpT3+iY8eO3Hzzzaxfvx4zA6BLly7s3LmTZs2akZubS7NmzYqtQ1ZWFitXruSRRx5h1KhRgfTLL7+c3/3ud9x///3MmDEjqvNZuHAhqamp1KhRg0svvbTYgFWrVi0WLlxIRkZGSPqQIUPo2LEjd9xxB9deey01aiTPnaHkqamISBl07do1JFgV6t+/PwBr164F4MCBA8yZM4fMzMxAsAI4/vjjGTx4MBs3bmTZsmWB9IYNG5YYpIItWLAAoMjqv2effTbt2rXjtddeIz8/P6qy0tLSogoytWrVKhKsAE488UQyMjLYuXMnO3fujOqYlYUClohUS4VXJieeeCIAn332GQUFBfTo0aNI3u7duwOEBKxYFBQUAFC/fv0i2+rXr8+BAwdYs6Z89+pisW3bNurUqUOTJk2O2THjQQFLRKqdI0eO8Le//Y1atWpx7bXXArBjxw6AiAMgCtO2b99epuN17NgRgPnz54ekf/vtt6xfvx6Ab775pkxlxyo7O5ulS5fSv39/6tWrd0yOGS8KWCJS7YwYMYJPPvmEMWPG8Mtf/hKAvLw8AOrWrVskf+Ev9sI8sRo2bBj169dn2LBhzJw5ky1btrB48WKuuOIKjhw5Uq6yY7Fp0yYGDhxIixYtmDBhQoUfL94UsESkWnnggQeYOnUqQ4cODRkAUdhdV9h9F6zw/lKkLr1otG3blrlz59KgQQOuvvpq0tLSyMjI4JRTTmHw4MEANGrUCPCGv3/33Xchr0OHDpXpuME2b97M+eefj5nxzjvvRH3/rTKpsgHLzJ4xs2/MbK+ZbTeziWZWJ9H1EpHEGT16NGPHjuWGG27gueeeC9nWvHlzIHK3X2FaLM9LhcvMzGTTpk18/vnnLFq0iK1btzJr1ixyc3MBaN++PQBPPPEEJ598csjrf/7nf8p8XICcnBzOPfdc9u/fz3/+8x86dapcz9pFqyoPa58K3OmcO2BmKcAbwL3A6ITWSkQSYvTo0Tz00EMMGjSIGTNmBIanF+rUqRN169blk08+KbLvkiVLAG/EYXmYGR06dAh8LigoYP78+bRt25bTTjsN8IbR/+53vwvZ74wzzijzMXNycsjMzGTPnj188MEH/PrXvy5zWYlWZQOWc+6LoI8GHAXaJag6IpJAY8aM4aGHHmLgwIE8//zzEYeFH3/88Vx22WXMnj2b1atXB4LE/v37mTFjBu3atePMM8+Ma73uvfdedu3axRNPPBFIO/XUUzn11FPjUv6WLVs499xz+fHHH/nPf/5Dly5d4lJuoiQ8YJnZKOA3QBegNbDFOZdWTN4awHDgz0Aa8D3wOvBX59yBCPnvAe4HGgC7gHvifwYiUpk9/fTTPPjgg6SmptKrVy9effXVkO0nnngiF1xwAeDNIjFv3jwuvPBCRo4cSaNGjZg+fTrbt29n7ty5Ra7Kxo4dC/w8YOKzzz4LpJ1zzjmcc845gbxdunTh3HPPpV27dhQUFPCvf/2LBQsWMHToUK6//vqoz+ff//43q1evBuDLL78MqUeTJk0CD0Hv27ePc889l5ycHG699VY2bNjAhg0bQsq64IILAsP6k0HCAxbwCLAbWAk0KSXvU8BtwH8DE4DT/c+/NrNezrmjwZmdc+OAcWZ2OnAd8G18qy4ilV3hs1Nbt25l0KBBRbZnZGQEAlbbtm35+OOPueeeexg3blxgLsF33303ZFqmQg888EDI51WrVrFq1SoAHnzwwZCA1aNHD+bMmcO2bduoVasW6enpvPrqq1xzzTUxnc+sWbP4xz/+EbEerVq1CgSsXbt2sXnzZgCmTJkSsawFCxYkVcCySJMmHtMKmJ3qnPva/3ktcHykKywz6wisAf7bOfeHoPRbgcnAdc65V8P3C8p3JXCTc+7ckurTtWtXt3z58jKdS1knDV2zeWuZ9iuUlv8qOeP6lKsMqV7WrVvH6aefnuhqSJKI5vtiZiucc+W7yVeKmEYJmtnXZnZ5CdsvNbOvYymzMFhF4Rq8e1ETw9KnA3lAabM41gZOi6VuIiJSecQ6rD0NOL6E7Q2AVmWuTcm64Q2cWBqc6JzLBz71twNgZo3N7Hoza2Keznj3st6roLqJiEgFi/dzWCfiXe1UhOZArnOu6FN9sB1ICXrOyuFdcX0N7AP+BWQDt1ZQ3UREpIKVOujCzM4BMoOS+plZ2whZTwCuxrvaqQj1gUjBCiA/KM8h59xeoOgd0mKY2VBgKEBqamp56igiIhUkmlGC5wIP+j87oJ//iuRLYGQc6hVJHvBfxWyrF5QnZs65acA08AZdlKUMERGpWNEErInAC3gDHr4GRgBvheVxwH7n3O441i3cDqCDmdWN0C3YAq+7sPwTbomISKVUasByzu0B9gCY2bnAOudcIlb9WgZcCJwJfFiYaGb1gHRgcQLqJJK0nHNFHoQVCZfoR5+CxTTowjm3KEHBCmAm3pXciLD0IXj3rl451hUSSVY1a9bk8OHDia6GJIHDhw9Ts2bNRFcDKMNMF2aWijc1UjvgF3hdhcGcc+78GMobyM9D4ZsBdczsfv/zFufcS36ha8zsaeAWM5uNN+qvcKaLRUCxDw2LSKiGDRuyd+9eUlJSEl0VqeT27t1Lw4YNE10NIMaAZWaX4E2LVAfYjzc/X3ndCGSEpf3Nf18EvBSUPgLIwRvR1wfIBabgzSUYMi2TiBTvhBNOYOtWb4aVRo0aUbt2bXUPSoBzjsOHD7N3715++OGHSjN6OtYrrEfxgsQVzrmyzV8UxjmXGUPeI3hzCCbfUpkilUjdunVJTU1l9+7d5OTkBFa9FSlUs2ZNGjZsSGpqasRVmBMh1oDVHrg/XsFKRBKnbt26gQUCRZJBrDNdfA9o6LiIiBxzsQasl4A/lJpLREQkzmLtEnwBONfM3gImAZuBIp3fzrnyrZchIiISJtaAtR7vWSgDLi0hX+UYtC8iIlVGrAFrDF7AEhEROaZiCljOudEVVA8REZESxXs9LBERkQoR60wX50STzzmniWhFRCSuYr2HtZDo7mFp0EUMOrUu27QnazZrMKaIVB+xBqwbiimjDXA93jx/WeWrkoiISFGxDrr4R3HbzOxxYGW5ayQiIhJB3AZdOOd+AGYAd8WrTBERkULxHiX4A3BqnMsUERGJX8Dyl6ofCHwXrzJFREQKxTqs/fliNp0A9MBbMfjO8lZKREQkXKyjBK8vJn03sBEY6ZzTUvUiIhJ3sY4S1MwYIiKSEApAIiKSFGLtEgTAzBoBvfh5RODXwH+cc/viVTEREZFgMQcsMxsMTACOx1sXC7zpmvab2e3Oub/HsX4iIiJA7KMELwem4V1RPQB87m/qCNwKTDOznc65f8e1llKinHrXwuhjfNDRe47xAUWkuov1CusuYB1wlnNuf1D6PDP7/4ElwN2AApaIiMRVrAHrDGBMWLACwDm3z8z+gXflJVVc2j1zAz/njOuTwJqISHUR6yhBK2V7NEuPiIiIxCzWgLUauN7MGoRvMLPj8R4sXh2HeomIiISItUvwcWA2sNLMJgNf+OmFgy7aAv3iVz0pSVkWftSijyKSrGKd6eJfZnYL8BgwhZ+7AA04ANzinHsrvlUUEREpw3NYzrlnzOxV4AKgtZ9c+OCwxjqLiEiFKNNMF865H4E34lsVERGR4pU66MLMaprZODP7Syn5hpnZI2ZW2khCERGRmEUzSnAA3hpXy0rJtxTvoeFrylspERGRcNEErKuAD5xzK0rK5G9/DwUsERGpANEErC7AB1GWtwDoWvbqiIiIRBbNoIsTgJ1Rlve9n1+qkeBpmo4lTQklUr1Ec4W1D0iJsrxfAEXmGRQRESmvaK6wPgcuxFsDqzQX8POSI1KF5dS79pgfMy3/1WN+TBGpPKK5wpoN9DKzviVl8tfKugCYFY+KiYiIBIsmYGUBXwKvm9nDZpYWvNHM0sxsLPA6sNHPLyIiEleldgk65w6aWR/gbWAUcI+Z7cW7t9UQaIQ3l+AG4FLnXH4F1ldERKqpqJYXcc59CaQDw4GPgCPASf77h376b5xzX1VMNUVEpLqLei5B/8ppiv8SERE5pmJdwFFERCQhFLBERCQplGl5EUleZVmlGLRSsYgknq6wREQkKShgiYhIUlDAEhGRpKCAJSIiSUEBS0REkoICloiIJAUFLBERSQoKWCIikhQUsEREJCkoYImISFJQwBIRkaSggCUiIklBAUtERJKCApaIiCQFBSwREUkKWg9LklbaPXMTduyccX0SdmyR6koBS6JSloUf473oY069a+NaXjTS8l895scUkcjUJSgiIkmhygYsM7vKzD4ys/1mlpPo+oiISPlU2YAF/ABMBe5LdEVERKT8quw9LOfcfwDM7IoEV0VEROIg4VdYZjbKzN4ws6/NzJXUfWdmNcxspJmtN7N8M/vGzCaYWYNjWGUREUmAhAcs4BHgPOArvG68kjwFPAl8AdwKvAHcBvzbzCrDuYiISAWpDF2CbZxzXwOY2Vrg+EiZzKwjXpCa7Zz7Q1D6ZmAycDWgMcgSV8UOpR9dgQcdvacCCxdJXgkPWIXBKgrXAAZMDEufDowDBqCAJVVASQ9E64Flqc4SHrBi0A04CiwNTnTO5ZvZp/72ADOrCdT2X2Zm9bzsruDYVFdEROIpme77NAdyiwk424EUM6sTlDYQOAi8DqT6P2+IVLCZDTWz5Wa2/Pvvv49ztUVEJB6SKWDVB4q7OsoPygOAc+4F55yFvdIi7eycm+ac6+qc69qsWbP41lpEROIimQJWHlC3mG31gvKIiEgVlEwBawdet1+koNUCr7vw0DGuk4iIHCPJNOhiGXAhcCbwYWGiP5giHVicmGqJHDuJWlJFoxOlMkimK6yZgANGhKUPwbt39cqxrpCIiBw7Cb/CMrOBQCv/YzOgjpnd73/e4px7CcA5t8bMngZuMbPZQDZwOt5MF4vQM1iVTlnW0IKyr6NVGdbsEpGKk/CABdwIZISl/c1/XwS8FJQ+AsgBhgJ9gFxgCvBX59zRCq2liIgkVMIDlnMuM4a8R4AJ/ktERKqRhAcskXBl7UqsKoqdv7ACpeWrR10qv2QadCEiItWYApaIiCQFBSwREUkKClgiIpIUFLBERCQpKGCJiEhS0LB2EUmM0Y0TcMw9x/6YEje6whIRkaSggCUiIklBAUtERJKCApaIiCQFDboQEYlAi2VWPrrCEhGRpKCAJSIiSUEBS0REkoICloiIJAUFLBERSQoKWCIikhQ0rF2kDDq1To15nzWbt1ZATeIjp961JWcYfUyqUeESNVRd4kNXWCIikhQUsEREJCkoYImISFJQwBIRkaSggCUiIklBAUtERJKCApaIiCQFBSwREUkKClgiIpIUFLBERCQpKGCJiEhS0FyCIiIVqNR5GsONLv8x0/JfLdN+OeP6lP/gFUhXWCIikhQUsEREJCkoYImISFJQwBIRkaSggCUiIklBAUtERJKCApaIiCQFBSwREUkKClgiIpIUzDmX6DpUKmb2PbCljLunALlxrE51oDaLndosNmqv2JWlzVo555pVRGUKKWDFkZktd851TXQ9konaLHZqs9iovWJXWdtMXYIiIpIUFLBERCQpKGDF17REVyAJqc1ipzaLjdordpWyzXQPS0REkoKusEREJCkoYImISFJQwBIRkaRQZQOWmdUws5Fmtt7M8s3sGzObYGYNKmJ/M+ttZv9jZgfMbLeZvWFmrYvJ+0sz+5eZ/eDn/9DMzismb2Mzm2Jm2/16fG5mw8zMom+N6FSFNjOzTDNzxbzejq1F4n/O5d0/2jYzszPNbLKZfWxm+/3zv76EetQ1szFmttnMCszsKzO738xqR90YUagK7WVmaSV8x9bG1CBRqIxtZp4BZvaamX1pZnlmttXM5pjZWRVxHgA456rkC5gEOGA2MAR4EjgMzAdqxHN/oB9wFFgF3ASMAv4P2AE0D8vbBtjlbx/l51/ll90rLG8dYKm/7Um/HrP9eo1Wm0Vss0y/DlnAgLBXZjVqs9HAEeBz4GP/GNeXUI9/+Xn+Dgz23x3wgtqrSB3SguoQ/h27tDp8x4B6fpmrgLHAjcD9wDZ//wHxPg/nXNUMWEBHv9FmhaXf6jfYtfHaH6gNbMebzun4oPR0/z/AtLAyXvfT04PSjvf334A/ctNPv8k/3q1hZcwCDuFNhaI2C22zTEr55VxN2uxEoIH/8x9LahOgt799Qlj6BD/9bLVXSN40KuiPxmRpM6AWkBHheCfiTen0fwQFofKeRyB/RTd4Il54Ed8BPcPS6wEHgOx47Q/08vM+EKGcecAeoLb/uQGQD8yLkPcBv5wzg9I+8o9XLyxvTz/vXWqzIm2WWfjLxt+3Xkn1roptFmF7ab+AX/a3twxLb+mnP6P2Ctme5m8f7R+7fnX/joXlneWXc1K8zqPwVVXvYXXDi+ZLgxOdc/nAp/72eO1f+PMnEcpZAjQCTvM/dwbqlpA3UJ6Z1QB+A6zyjxtsKd4/fmnnEYukb7Mwk4D9wEEz22hmw83ift+vsrZZrLoB251z34TV4xu8rqB4fc+qSnsVugPIAw7492PGmFndcpYZLhnb7BS8HqAfy1iPYlXVgNUcyHXOFUTYth1IMbM6cdq/eVB6pLwALcqQtylwXKS8fr1yg/LGQ1VoM/D6xOcAdwGXA3/B+48zEXi+hPqXRWVts1g1L6bcwrLj9T2rKu11FO++y73AFXj3/L7Au+J/28xqlrHcSJKqzcysN3AmMDPsD+3yngfg9UNWRfWBSA0DXvdSYZ5Dcdi/vv85Uv7gvMQxb2H++sVsK4uq0GY45z4G+gZnMrPpQDZwvZnN8PPEQ2Vts1iVVo94fc+qRHs557YC54cl/93MpuENJrgaeKUsZUeQNG1mZu2Al/AC0B3lqEexquoVVh5eN1Ik9YLyxGP/wvdI+Ssqb2H+ks4hVlWhzSJyzh0FHvU/9ikpb4wqa5vFqrR6xOt7VlXaqzgP++/V7jvmD3ufh3er4hLn3PflqEexqmrA2oF3iRmpgVrgXZqWFMlj2X9HUHqkvPDz5XQseX8ADkbK69crheK7ccqiKrRZSXL895Qo8karsrZZrHYUU25h2fH6nlWV9irON3ij6arVd8zM0oAFeCN3L3DOrSlnPYpVVQPWMrxzOzM40czq4Q3RXB7H/Zf57z0ilNMd2Ats9D+vwbssLi4vhWX7VwUrgV9H+Ec+E7AoziMWSd9mpWjnv/9fFHmjVVnbLFbLgBZm1jKsHi3x7j3E63tWVdqrOKcCNalG3zE/WC0EGuMFq1UVdB6eihqOmcgX0ImSx/wPCEprA7Qvx/618f56CH924Qy8v7ZmhJXxhp9+RlBa4TNFGwl9puhmin8O6zCQpjYr0ma/iHBudfEeEQgZAl+V2yysvNKGafeh5Oewfqf2Ctke6TtWA3jN3++q6vAdA1oBm/EGNXWL13mUWE68GrayvYAp/PxU9WD/P99hvL8Ggh9oywFcWff3815J6NPh9+D9lfUd0CIsb1tgt7/9Hn6eteEn4KKwvHXw/vI47B9/MD/PdPE3tVnENlsGvAX81a/DX/GCmgMmV6M2a4U388D9wD/9Y8wKSmsVlv/ffp4ZeLMWzPA/v6T2Cm0v/9gfAGPwBlmMwvt/6vBmDIlq1oZkbjOgIfC1X+5kis74MQA4saz1KLYt4v0fuLK88C7N78CbCaEAr+/1SYL+cijlHzmq/YPyX4r3rEIe3v2nN4E2xeQ9He+X6o9+/o8Im2IoKG8TYCreXz4FeMNnbyHoqkJtFpLvbrznSL73/zP8iNe/fk11+p7x8wPUxb0yw/LXw3u4M8evx9d4w7RLfVC0urUXXkBfiPdL/BCwzz/GTcQ5WFXWNuPnh6dj+Y7FVI9ILy3gKCIiSaGqDroQEZEqRgFLRESSggKWiIgkBQUsERFJCgpYIiKSFBSwREQkKShgiYhIUlDAEpFyMbNMM3Nmdn2i6yJVmwKWVDtBv2CDX/vNbIW/MnE8F+CrMGa2MOwcDpvZDjObaWa/SnT9ROKtqi7gKBKNf+It7Gh4s5Jfj7cycUdgaMJqFZsCvHnZwFuhugtwA9DbzLo65zYkrGYicaaAJdXZSufcy4UfzOxZYB0w2MwecM7Fc5mIivJT8DkA083sC2AS3pyTtyamWiLxpy5BEZ9zbi/exLkGnGpmNczsPjNbbGbfmdkhM9tqZs+a2S/C9zez+mb2pJl9a2YHzWyJmZ1vZi+YWZFJO82snZm95Oc/ZGY5Zva4mTUo56nM89/bBSea2Zl+XTaaWZ6Z7TOzj83s9xHq9oLfzdjYP9+dZpbv5z8rmkqY2SC/m/JNf90jkXLRFZaIz8wMbykTgFy85V3uxFtm4i3gANANb7bu35lZFxe6SuobQG+8JSY+AFoD/423ZlD4sboA8/Fmk8/Cm7n6DOA24LdmluGcO1zGU2njv+8OS/890B54HW/No18Ag4DZZnadc+7VCGW9hzfz/Rg//+3AXDNr7ZzbV1wFzOxevCXjnwZuc96CpCLlE++p8PXSq7K/+Hkpib/iLWfeDOgMTPfTP/HzGXBchP1vJGyhPrxA5YDpYXkL011Y+mpgPdAwLP33lLB4YFjehcB+/xxSgJbAFfjLTAC9w/I3iFBGfbzlHr4IS3/BL+OZsPQr/fQ/R2jP6/F6bZ72P9+b6H9rvarWS12CUp09hHf1sBMvgPx/wBy8X/o4z0EAM6tpZk3MLAXvyggguGvsMv/9yeADOOey8e6LBZhZJ7wA+SpQ18xSCl9463wdAC6M8hwa+OfwPbAV74quDjDIP3ZwXQ4E1aG+361Z3z+f082sUYTynwr7XHju7cIz4q2p9SbegJXrnXOPRHkOIlFRl6BUZ9PwuvEcXpDY6JwL6UYzs6vwFp37Nd4S4sGaBv3cGm+l1i8jHGcD3gKUhQp/fsh/RXJiFPUHyOfnYHkC8CfgAiLcnzaz/8JbpLEv8F8RymoC7A1L+zr4g3Nul9dzSpF7eMB4vJVoi+teFCkXBSypzjY55z4obqOZ9QNmAkuB4cA3eAGiJvAukQctRbMiqvnvE/xyIvkhinIAjgSfg5m9CbwNTDOzlc65z/x0A97HC5aT8JZ03wMcwRsGfy0Rzsc5d6SUcwj2L+APwJ1m9p5zbleU5yASFQUskeINxAtQ5zrn8goTzax9hLw5eL/w2xHWBQj8MuzzJv/9SEkBsyycc0fNbDjwBfAEP3ctdsYb1DHGOfdg8D5mNpj4mA88jxcwF5hZL+fczjiVLaJ7WCIlOIJ3xRT4f+JfqdwfIe+//feRwYlm1pvQ7kCAVcBa4C9mdmp4QWZWy8xOKGulnXOb8O6PXWBmv/OTC6+UQq6M/BkxigxrL8exFwIXA2l4QeukeJUtoisskeK9idfFNd/MXsS7h3UF3kCFcNl4Q8CH+IMnCoe1DwU+w7vCAbzBHGY2EO+K5DMzex743C+3LdAPGIU3Uq+sHgEG4N0jOx/vqu9z4C4zKxwZeBrwZ2AN3gwZceGc+8jMLsTr7lxoZuc553bEq3ypvnSFJVIM59xreAHneLzutbvwftFfFCGvwwtuk4Df4t2fOgfv6mUTcDAs/6d4AzleBi4HpuBduXXHC1TzKAfnTcn0OnCe/0zXEaAP3pXgIL+eGf7Pb5fnWMUcfwnQC29wxyIzaxnvY0j1Y97/MxGpKGa2BqjtnIt070tEoqQrLJE4MbPjIqT1AX4F/OfY10ikatEVlkicmNmjeN18C/CGjKfjPYy8F0h3zm1LXO1Ekp8Clkic+CMC7wE6AI3x5vKbDzzgnIv0QLGIxEABS0REkoLuYYmISFJQwBIRkaSggCUiIklBAUtERJKCApaIiCSF/wcwcQ7jlypvbAAAAABJRU5ErkJggg==",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.hist(df_1980_01[\"PR_1980_01\"], label=\"1980-01\")\n",
    "plt.hist(df_1980_02[\"PR_1980_02\"], label=\"1980-02\")\n",
    "plt.hist(df_2019_12[\"PR_2019_12\"], label=\"2019-12\")\n",
    "\n",
    "plt.yscale(\"log\")\n",
    "\n",
    "plt.legend()\n",
    "plt.xlabel(\"Page Rank\")\n",
    "plt.ylabel(\"Count\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                     ID  PR_1980_01  PR_1980_02  PR_2019_11  PR_2019_12\n",
      "0       SCHEMBL11422812    0.000030         NaN         NaN         NaN\n",
      "1       SCHEMBL11488819    0.000029         NaN         NaN         NaN\n",
      "2        SCHEMBL9429923    0.000051         NaN         NaN         NaN\n",
      "3       SCHEMBL11245945    0.000019         NaN         NaN         NaN\n",
      "4       SCHEMBL11486700    0.000034         NaN         NaN         NaN\n",
      "...                 ...         ...         ...         ...         ...\n",
      "352703  SCHEMBL21584018         NaN         NaN         NaN    0.000010\n",
      "352704  SCHEMBL18147995         NaN         NaN         NaN    0.000013\n",
      "352705  SCHEMBL21599701         NaN         NaN         NaN    0.000013\n",
      "352706  SCHEMBL21602312         NaN         NaN         NaN    0.000010\n",
      "352707  SCHEMBL21600585         NaN         NaN         NaN    0.000007\n",
      "\n",
      "[352708 rows x 5 columns]\n"
     ]
    }
   ],
   "source": [
    "#Testing merging dataframes (its working!)\n",
    "\n",
    "dataframes = [df_1980_01, df_1980_02, df_2019_11, df_2019_12]\n",
    "\n",
    "df = reduce(lambda left, right: pd.merge(left, right, on=[\"ID\"], how=\"outer\"), dataframes)\n",
    "\n",
    "print(df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Tidy data\n",
    "\n",
    "Get all pagerank values for each compound, put them in a dataframe with each unique ID as a row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def build_month_list(start, end):\n",
    "    \"\"\" Builds a list of all months in a given range\n",
    "\n",
    "    Args:\n",
    "        start (int): year describing the start of the data\n",
    "        end (int): year describing the end of the data (inclusive)\n",
    "\n",
    "    Returns:\n",
    "        list: list of all update months in format \"YYYY-MM\"\n",
    "    \"\"\"\n",
    "    updates = []\n",
    "    for year in range(start, end + 1):  # all years through the given end\n",
    "        for month in range(1, 13):  #include 12 months\n",
    "            if month < 10:\n",
    "                updates.append(str(year) + \"-0\" + str(month))\n",
    "            else:\n",
    "                updates.append(str(year) + \"-\" + str(month))\n",
    "\n",
    "    return updates"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading dataframes...\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 480/480 [04:52<00:00,  1.64it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Merging dataframes...\n"
     ]
    },
    {
     "ename": "MemoryError",
     "evalue": "Unable to allocate 132. MiB for an array with shape (8661923,) and data type complex128",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mMemoryError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[1;32mz:\\group\\John Malloy\\Projects\\Patents\\pageRank.ipynb Cell 10\u001b[0m in \u001b[0;36m<cell line: 10>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m     dataframes\u001b[39m.\u001b[39mappend(pd\u001b[39m.\u001b[39mDataFrame(pickle\u001b[39m.\u001b[39mload(file\u001b[39m=\u001b[39m\u001b[39mopen\u001b[39m(fp \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mpagerank_\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m+\u001b[39m \u001b[39mstr\u001b[39m(month) \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m.p\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mrb\u001b[39m\u001b[39m\"\u001b[39m)), \n\u001b[0;32m      <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m         columns\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39mID\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mstr\u001b[39m(month)]))\n\u001b[0;32m      <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mMerging dataframes...\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m df \u001b[39m=\u001b[39m reduce(\u001b[39mlambda\u001b[39;49;00m left, right: pd\u001b[39m.\u001b[39;49mmerge(left, right, on\u001b[39m=\u001b[39;49m[\u001b[39m\"\u001b[39;49m\u001b[39mID\u001b[39;49m\u001b[39m\"\u001b[39;49m], how\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mouter\u001b[39;49m\u001b[39m\"\u001b[39;49m), dataframes)\n\u001b[0;32m     <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39mprint\u001b[39m(df)\n\u001b[0;32m     <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m fp \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mG:Shared drives/SureChemBL_Patents/PageRank/\u001b[39m\u001b[39m\"\u001b[39m\n",
      "\u001b[1;32mz:\\group\\John Malloy\\Projects\\Patents\\pageRank.ipynb Cell 10\u001b[0m in \u001b[0;36m<lambda>\u001b[1;34m(left, right)\u001b[0m\n\u001b[0;32m      <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m     dataframes\u001b[39m.\u001b[39mappend(pd\u001b[39m.\u001b[39mDataFrame(pickle\u001b[39m.\u001b[39mload(file\u001b[39m=\u001b[39m\u001b[39mopen\u001b[39m(fp \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mpagerank_\u001b[39m\u001b[39m\"\u001b[39m \u001b[39m+\u001b[39m \u001b[39mstr\u001b[39m(month) \u001b[39m+\u001b[39m \u001b[39m\"\u001b[39m\u001b[39m.p\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mrb\u001b[39m\u001b[39m\"\u001b[39m)), \n\u001b[0;32m      <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=6'>7</a>\u001b[0m         columns\u001b[39m=\u001b[39m[\u001b[39m\"\u001b[39m\u001b[39mID\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39mstr\u001b[39m(month)]))\n\u001b[0;32m      <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m \u001b[39mprint\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39mMerging dataframes...\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[1;32m---> <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=9'>10</a>\u001b[0m df \u001b[39m=\u001b[39m reduce(\u001b[39mlambda\u001b[39;00m left, right: pd\u001b[39m.\u001b[39;49mmerge(left, right, on\u001b[39m=\u001b[39;49m[\u001b[39m\"\u001b[39;49m\u001b[39mID\u001b[39;49m\u001b[39m\"\u001b[39;49m], how\u001b[39m=\u001b[39;49m\u001b[39m\"\u001b[39;49m\u001b[39mouter\u001b[39;49m\u001b[39m\"\u001b[39;49m), dataframes)\n\u001b[0;32m     <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m \u001b[39mprint\u001b[39m(df)\n\u001b[0;32m     <a href='vscode-notebook-cell:/z%3A/group/John%20Malloy/Projects/Patents/pageRank.ipynb#X24sZmlsZQ%3D%3D?line=13'>14</a>\u001b[0m fp \u001b[39m=\u001b[39m \u001b[39m\"\u001b[39m\u001b[39mG:Shared drives/SureChemBL_Patents/PageRank/\u001b[39m\u001b[39m\"\u001b[39m\n",
      "File \u001b[1;32mc:\\Users\\group\\miniconda3\\lib\\site-packages\\pandas\\core\\reshape\\merge.py:122\u001b[0m, in \u001b[0;36mmerge\u001b[1;34m(left, right, how, on, left_on, right_on, left_index, right_index, sort, suffixes, copy, indicator, validate)\u001b[0m\n\u001b[0;32m     90\u001b[0m \u001b[39m@Substitution\u001b[39m(\u001b[39m\"\u001b[39m\u001b[39m\\n\u001b[39;00m\u001b[39mleft : DataFrame or named Series\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m     91\u001b[0m \u001b[39m@Appender\u001b[39m(_merge_doc, indents\u001b[39m=\u001b[39m\u001b[39m0\u001b[39m)\n\u001b[0;32m     92\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mmerge\u001b[39m(\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    105\u001b[0m     validate: \u001b[39mstr\u001b[39m \u001b[39m|\u001b[39m \u001b[39mNone\u001b[39;00m \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m,\n\u001b[0;32m    106\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m DataFrame:\n\u001b[0;32m    107\u001b[0m     op \u001b[39m=\u001b[39m _MergeOperation(\n\u001b[0;32m    108\u001b[0m         left,\n\u001b[0;32m    109\u001b[0m         right,\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    120\u001b[0m         validate\u001b[39m=\u001b[39mvalidate,\n\u001b[0;32m    121\u001b[0m     )\n\u001b[1;32m--> 122\u001b[0m     \u001b[39mreturn\u001b[39;00m op\u001b[39m.\u001b[39;49mget_result()\n",
      "File \u001b[1;32mc:\\Users\\group\\miniconda3\\lib\\site-packages\\pandas\\core\\reshape\\merge.py:738\u001b[0m, in \u001b[0;36m_MergeOperation.get_result\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    735\u001b[0m \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mindicator:\n\u001b[0;32m    736\u001b[0m     result \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_indicator_post_merge(result)\n\u001b[1;32m--> 738\u001b[0m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49m_maybe_add_join_keys(result, left_indexer, right_indexer)\n\u001b[0;32m    740\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_maybe_restore_index_levels(result)\n\u001b[0;32m    742\u001b[0m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_maybe_drop_cross_column(result, \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_cross)\n",
      "File \u001b[1;32mc:\\Users\\group\\miniconda3\\lib\\site-packages\\pandas\\core\\reshape\\merge.py:916\u001b[0m, in \u001b[0;36m_MergeOperation._maybe_add_join_keys\u001b[1;34m(self, result, left_indexer, right_indexer)\u001b[0m\n\u001b[0;32m    914\u001b[0m     result_dtype \u001b[39m=\u001b[39m lvals\u001b[39m.\u001b[39mdtype\n\u001b[0;32m    915\u001b[0m \u001b[39melse\u001b[39;00m:\n\u001b[1;32m--> 916\u001b[0m     key_col \u001b[39m=\u001b[39m Index(lvals)\u001b[39m.\u001b[39mwhere(\u001b[39m~\u001b[39mmask_left, rvals)\n\u001b[0;32m    917\u001b[0m     result_dtype \u001b[39m=\u001b[39m find_common_type([lvals\u001b[39m.\u001b[39mdtype, rvals\u001b[39m.\u001b[39mdtype])\n\u001b[0;32m    919\u001b[0m \u001b[39mif\u001b[39;00m result\u001b[39m.\u001b[39m_is_label_reference(name):\n",
      "File \u001b[1;32mc:\\Users\\group\\miniconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py:494\u001b[0m, in \u001b[0;36mIndex.__new__\u001b[1;34m(cls, data, dtype, copy, name, tupleize_cols, **kwargs)\u001b[0m\n\u001b[0;32m    491\u001b[0m arr \u001b[39m=\u001b[39m com\u001b[39m.\u001b[39masarray_tuplesafe(data, dtype\u001b[39m=\u001b[39m_dtype_obj)\n\u001b[0;32m    493\u001b[0m \u001b[39mif\u001b[39;00m dtype \u001b[39mis\u001b[39;00m \u001b[39mNone\u001b[39;00m:\n\u001b[1;32m--> 494\u001b[0m     arr \u001b[39m=\u001b[39m _maybe_cast_data_without_dtype(\n\u001b[0;32m    495\u001b[0m         arr, cast_numeric_deprecated\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m\n\u001b[0;32m    496\u001b[0m     )\n\u001b[0;32m    497\u001b[0m     dtype \u001b[39m=\u001b[39m arr\u001b[39m.\u001b[39mdtype\n\u001b[0;32m    499\u001b[0m     \u001b[39mif\u001b[39;00m kwargs:\n",
      "File \u001b[1;32mc:\\Users\\group\\miniconda3\\lib\\site-packages\\pandas\\core\\indexes\\base.py:7124\u001b[0m, in \u001b[0;36m_maybe_cast_data_without_dtype\u001b[1;34m(subarr, cast_numeric_deprecated)\u001b[0m\n\u001b[0;32m   7106\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39m_maybe_cast_data_without_dtype\u001b[39m(\n\u001b[0;32m   7107\u001b[0m     subarr: np\u001b[39m.\u001b[39mndarray, cast_numeric_deprecated: \u001b[39mbool\u001b[39m \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m\n\u001b[0;32m   7108\u001b[0m ) \u001b[39m-\u001b[39m\u001b[39m>\u001b[39m ArrayLike:\n\u001b[0;32m   7109\u001b[0m     \u001b[39m\"\"\"\u001b[39;00m\n\u001b[0;32m   7110\u001b[0m \u001b[39m    If we have an arraylike input but no passed dtype, try to infer\u001b[39;00m\n\u001b[0;32m   7111\u001b[0m \u001b[39m    a supported dtype.\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m   7121\u001b[0m \u001b[39m    np.ndarray or ExtensionArray\u001b[39;00m\n\u001b[0;32m   7122\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m-> 7124\u001b[0m     result \u001b[39m=\u001b[39m lib\u001b[39m.\u001b[39;49mmaybe_convert_objects(\n\u001b[0;32m   7125\u001b[0m         subarr,\n\u001b[0;32m   7126\u001b[0m         convert_datetime\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m   7127\u001b[0m         convert_timedelta\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m   7128\u001b[0m         convert_period\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m   7129\u001b[0m         convert_interval\u001b[39m=\u001b[39;49m\u001b[39mTrue\u001b[39;49;00m,\n\u001b[0;32m   7130\u001b[0m         dtype_if_all_nat\u001b[39m=\u001b[39;49mnp\u001b[39m.\u001b[39;49mdtype(\u001b[39m\"\u001b[39;49m\u001b[39mdatetime64[ns]\u001b[39;49m\u001b[39m\"\u001b[39;49m),\n\u001b[0;32m   7131\u001b[0m     )\n\u001b[0;32m   7132\u001b[0m     \u001b[39mif\u001b[39;00m result\u001b[39m.\u001b[39mdtype\u001b[39m.\u001b[39mkind \u001b[39min\u001b[39;00m [\u001b[39m\"\u001b[39m\u001b[39mi\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mu\u001b[39m\u001b[39m\"\u001b[39m, \u001b[39m\"\u001b[39m\u001b[39mf\u001b[39m\u001b[39m\"\u001b[39m]:\n\u001b[0;32m   7133\u001b[0m         \u001b[39mif\u001b[39;00m \u001b[39mnot\u001b[39;00m cast_numeric_deprecated:\n\u001b[0;32m   7134\u001b[0m             \u001b[39m# i.e. we started with a list, not an ndarray[object]\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\group\\miniconda3\\lib\\site-packages\\pandas\\_libs\\lib.pyx:2457\u001b[0m, in \u001b[0;36mpandas._libs.lib.maybe_convert_objects\u001b[1;34m()\u001b[0m\n",
      "\u001b[1;31mMemoryError\u001b[0m: Unable to allocate 132. MiB for an array with shape (8661923,) and data type complex128"
     ]
    }
   ],
   "source": [
    "months = build_month_list(1980, 2019)\n",
    "dataframes = []\n",
    "\n",
    "print(\"Loading dataframes...\")\n",
    "for month in tqdm(months):\n",
    "    dataframes.append(pd.DataFrame(pickle.load(file=open(fp + \"pagerank_\" + str(month) + \".p\", \"rb\")), \n",
    "        columns=[\"ID\", str(month)]))\n",
    "\n",
    "print(\"Merging dataframes...\")\n",
    "df = reduce(lambda left, right: pd.merge(left, right, on=[\"ID\"], how=\"outer\"), dataframes)\n",
    "\n",
    "print(df)\n",
    "\n",
    "fp = \"G:Shared drives/SureChemBL_Patents/PageRank/\"\n",
    "df.to_csv(fp + \"pageRank_data.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 ID   1980-01   1980-02   1980-03   1980-04   1980-05  \\\n",
      "13      SCHEMBL1748  0.000393  0.000362  0.000357  0.000299  0.000503   \n",
      "27     SCHEMBL23098  0.000040  0.000088  0.000051  0.000037  0.000155   \n",
      "40     SCHEMBL15681  0.000101  0.000019  0.000041  0.000115  0.000046   \n",
      "48     SCHEMBL51803  0.000089  0.000042  0.000030  0.000026  0.000044   \n",
      "56       SCHEMBL968  0.000342  0.000233  0.000374  0.000242  0.000278   \n",
      "...             ...       ...       ...       ...       ...       ...   \n",
      "26615  SCHEMBL80292  0.000381  0.000279  0.000233  0.000255  0.000247   \n",
      "26691  SCHEMBL22165  0.000056  0.000034  0.000053  0.000019  0.000104   \n",
      "26699  SCHEMBL19807  0.000056  0.000129  0.000045  0.000059  0.000025   \n",
      "26730  SCHEMBL14893  0.000206  0.000310  0.000283  0.000185  0.000255   \n",
      "26738  SCHEMBL15789  0.000127  0.000274  0.000208  0.000199  0.000317   \n",
      "\n",
      "        1980-06   1980-07   1980-08   1980-09  ...   1984-03   1984-04  \\\n",
      "13     0.000431  0.000377  0.000607  0.000363  ...  0.000328  0.000565   \n",
      "27     0.000030  0.000066  0.000060  0.000102  ...  0.000059  0.000051   \n",
      "40     0.000034  0.000065  0.000044  0.000064  ...  0.000116  0.000022   \n",
      "48     0.000062  0.000056  0.000133  0.000029  ...  0.000041  0.000110   \n",
      "56     0.000174  0.000147  0.000139  0.000162  ...  0.000209  0.000203   \n",
      "...         ...       ...       ...       ...  ...       ...       ...   \n",
      "26615  0.000359  0.000275  0.000388  0.000305  ...  0.000298  0.000308   \n",
      "26691  0.000085  0.000080  0.000093  0.000113  ...  0.000099  0.000048   \n",
      "26699  0.000136  0.000025  0.000112  0.000014  ...  0.000083  0.000135   \n",
      "26730  0.000239  0.000257  0.000172  0.000166  ...  0.000349  0.000330   \n",
      "26738  0.000143  0.000263  0.000175  0.000231  ...  0.000252  0.000125   \n",
      "\n",
      "        1984-05   1984-06   1984-07   1984-08   1984-09   1984-10   1984-11  \\\n",
      "13     0.000479  0.000404  0.000389  0.000482  0.000366  0.000431  0.000458   \n",
      "27     0.000083  0.000055  0.000049  0.000055  0.000088  0.000087  0.000094   \n",
      "40     0.000032  0.000070  0.000056  0.000101  0.000038  0.000087  0.000053   \n",
      "48     0.000066  0.000025  0.000044  0.000021  0.000065  0.000065  0.000061   \n",
      "56     0.000215  0.000279  0.000228  0.000197  0.000395  0.000260  0.000191   \n",
      "...         ...       ...       ...       ...       ...       ...       ...   \n",
      "26615  0.000418  0.000235  0.000214  0.000494  0.000345  0.000241  0.000464   \n",
      "26691  0.000118  0.000041  0.000140  0.000135  0.000122  0.000089  0.000057   \n",
      "26699  0.000066  0.000133  0.000137  0.000092  0.000087  0.000101  0.000049   \n",
      "26730  0.000310  0.000289  0.000243  0.000256  0.000306  0.000325  0.000298   \n",
      "26738  0.000188  0.000141  0.000112  0.000100  0.000134  0.000160  0.000210   \n",
      "\n",
      "        1984-12  \n",
      "13     0.000311  \n",
      "27     0.000049  \n",
      "40     0.000048  \n",
      "48     0.000037  \n",
      "56     0.000417  \n",
      "...         ...  \n",
      "26615  0.000205  \n",
      "26691  0.000143  \n",
      "26699  0.000044  \n",
      "26730  0.000195  \n",
      "26738  0.000161  \n",
      "\n",
      "[1107 rows x 61 columns]\n"
     ]
    }
   ],
   "source": [
    "df_noNA = df.dropna()\n",
    "\n",
    "print(df_noNA)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.9.12 ('base')",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "b6d33183bcf47bf2749a1bb5bc5cecc0497fbf4071239c16ea842776928851e7"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}