examples/Method_Examples/Example Novelty.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d43b5ebe",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "import pyscisci.all as pyscisci\n",
    "\n",
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "# set this path to where the MAG database will be stored\n",
    "path2mag = '/home/ajgates/MAG'\n",
    "\n",
    "mymag = pyscisci.MAG(path2mag, database_extension='hdf', keep_in_memory=False) "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f9fbffeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8fb4f28c6aff4d40af5f45731bdf71da",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading Publications:   0%|          | 0/132 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "PublicationId    29911736\n",
       "Year                   51\n",
       "DocType                 1\n",
       "JournalId           29507\n",
       "dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# first get all journal articles published from 1950 until 2001 \n",
    "pub = mymag.load_publications(columns=['PublicationId', 'Year', 'DocType', 'JournalId'], \n",
    "                              filter_dict = {'Year':np.arange(1950, 2001), 'DocType':np.sort(['j'])},\n",
    "                              duplicate_subset = ['PublicationId'],\n",
    "                              dropna = ['PublicationId', 'Year', 'DocType', 'JournalId']\n",
    "                             )\n",
    "\n",
    "pub.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "a467dfeb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0fe0cb75c9834d78a63dcdd84f5e6376",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/186 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CitingPublicationId    12311965\n",
       "CitedPublicationId     15220488\n",
       "dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "focus_publications = np.sort(pub['PublicationId'].unique())\n",
    "\n",
    "pub2ref = mymag.load_references(filter_dict = {'CitingPublicationId':focus_publications,\n",
    "                                               'CitedPublicationId':focus_publications})\n",
    "\n",
    "pub2ref.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6385454",
   "metadata": {},
   "outputs": [],
   "source": [
    "focus_publications = np.sort(np.unique(np.hstack([pub2ref['CitingPublicationId'].unique(), \n",
    "                                                  pub2ref['CitedPublicationId'].unique()])))\n",
    "pub = pub[pyscisci.isin_sorted(pub['PublicationId'].values, focus_publications)].reset_index(drop=True)\n",
    "pub.nunique()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "61b131cb",
   "metadata": {},
   "source": [
    "# Run the Novelty and Concentionality Calculation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "beb58c12",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "30a67037c87b474baf4c0491f0956998",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Novelty_Conventoinality:   0%|          | 0/51 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "\n",
    "novelty_score = pyscisci.novelty_conventionality(pub, pub2ref, focus_pub_ids=None, n_samples = 10, show_progress=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7f1d8d2f",
   "metadata": {},
   "outputs": [],
   "source": [
    "novelty_score.to_csv(os.path.join(path2mag, 'uzzi_novelty.csv.gz'), index=False, header=True, compression='gzip')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "47d9ee26",
   "metadata": {},
   "outputs": [],
   "source": [
    "novelty_score=pd.read_csv(os.path.join(path2mag, 'uzzi_novelty.csv.gz'), compression='gzip') "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "574bf0a7",
   "metadata": {},
   "source": [
    "# Find Hit papers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "75f9e180",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(89224594,)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# first load the publication doctype information\n",
    "pub2doctype = mymag.pub2doctype\n",
    "\n",
    "journal_pubs = np.sort([pid for pid,dt in pub2doctype.items() if dt=='j'])\n",
    "journal_pubs.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "02ac9f10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(29911736,)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# and then load the publication year information\n",
    "pub2year = mymag.pub2year\n",
    "\n",
    "journal_pub_1950_2000 = np.sort([pid for pid in journal_pubs if pub2year.get(pid, 0) >=1950 and pub2year.get(pid, 0) <=2000])\n",
    "journal_pub_1950_2000.shape\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "8fc4b07c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "71c4822a66e24affb367dc69dea3fa50",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading pub2ref:   0%|          | 0/186 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "CitingPublicationId    40282022\n",
       "CitedPublicationId     18549070\n",
       "dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# first get the reference list and merge with year info\n",
    "pub2ref = mymag.load_references(filter_dict = {'CitingPublicationId':journal_pubs,\n",
    "                                              'CitedPublicationId':journal_pub_1950_2000})\n",
    "pub2ref.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "898d7e4f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CitingPublicationId    40267558\n",
       "CitedPublicationId     18536578\n",
       "dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "# keep all citations that happend after or equal to the publication year\n",
    "pub2ref = pub2ref.loc[[pub2year.get(citingpid, -1) >= pub2year.get(citedpid, 50000) for citingpid, citedpid in pub2ref[['CitingPublicationId', 'CitedPublicationId']].values]]\n",
    "\n",
    "pub2ref.nunique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0da59580",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CitingPublicationId    18270919\n",
      "CitedPublicationId     15764952\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# keep only citations within the 8-year window used by Uzzi et al.\n",
    "cite_window = 8\n",
    "pub2ref = pub2ref.loc[[pub2year.get(citingpid, 0) <= pub2year.get(citedpid, 0) + cite_window for citingpid, citedpid in pub2ref[['CitingPublicationId', 'CitedPublicationId']].values]]\n",
    "print(pub2ref.nunique())\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "bbf9c9c2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PublicationId    15764952\n",
      "C8                   1656\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# calcuate the total citations\n",
    "citation = pyscisci.groupby_count(pub2ref, colgroupby='CitedPublicationId', colcountby='CitingPublicationId', count_unique=True )\n",
    "citation.rename(columns={'CitingPublicationIdCount':'C8', 'CitedPublicationId':'PublicationId'}, inplace=True)\n",
    "\n",
    "print(citation.nunique())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "888550f1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(14146784,)\n",
      "PublicationId    29911736\n",
      "C8                   1657\n",
      "Year                   51\n",
      "dtype: int64\n"
     ]
    }
   ],
   "source": [
    "# add back in the journal articles with zero citations\n",
    "zero_cites = journal_pub_1950_2000[~pyscisci.isin_sorted(journal_pub_1950_2000, np.sort(citation['PublicationId'].values))]\n",
    "print(zero_cites.shape)\n",
    "\n",
    "zero_cites = pd.DataFrame(np.vstack([zero_cites, np.zeros(zero_cites.shape)]).T, columns = ['PublicationId', 'C8'])\n",
    "\n",
    "citation = pd.concat([citation, zero_cites])\n",
    "citation['Year'] = [pub2year.get(pid, None) for pid in citation['PublicationId'].values]\n",
    "print(citation.nunique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2d25e1e5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>C8</th>\n",
       "      <th>Year</th>\n",
       "      <th>CitationRank</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.963551e+09</td>\n",
       "      <td>19.0</td>\n",
       "      <td>1995</td>\n",
       "      <td>0.895929</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.969463e+09</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2000</td>\n",
       "      <td>0.472650</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.974258e+09</td>\n",
       "      <td>15.0</td>\n",
       "      <td>1996</td>\n",
       "      <td>0.861436</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.998432e+09</td>\n",
       "      <td>48.0</td>\n",
       "      <td>1996</td>\n",
       "      <td>0.969797</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2.006096e+09</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1996</td>\n",
       "      <td>0.635578</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14146779</th>\n",
       "      <td>3.177495e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1985</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14146780</th>\n",
       "      <td>3.177496e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1959</td>\n",
       "      <td>0.499998</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14146781</th>\n",
       "      <td>3.177496e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1981</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14146782</th>\n",
       "      <td>3.177497e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1987</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14146783</th>\n",
       "      <td>3.177497e+09</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1963</td>\n",
       "      <td>0.499998</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>29911736 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          PublicationId    C8  Year  CitationRank\n",
       "0          1.963551e+09  19.0  1995      0.895929\n",
       "1          1.969463e+09   1.0  2000      0.472650\n",
       "2          1.974258e+09  15.0  1996      0.861436\n",
       "3          1.998432e+09  48.0  1996      0.969797\n",
       "4          2.006096e+09   4.0  1996      0.635578\n",
       "...                 ...   ...   ...           ...\n",
       "14146779   3.177495e+09   0.0  1985      0.000000\n",
       "14146780   3.177496e+09   0.0  1959      0.499998\n",
       "14146781   3.177496e+09   0.0  1981      0.000000\n",
       "14146782   3.177497e+09   0.0  1987      0.000000\n",
       "14146783   3.177497e+09   0.0  1963      0.499998\n",
       "\n",
       "[29911736 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# now get the citation rank for each year\n",
    "def myrank(a):\n",
    "    return pyscisci.rank_array(a, ascending=True, normed=True)\n",
    "citation['CitationRank'] = citation.groupby('Year')['C8'].transform(myrank)\n",
    "citation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0c92bb14",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Hit papers are those which appear in the top 5th percentile of the citations for that year\n",
    "citation['HitPaper'] = citation['CitationRank'] >= 0.95"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "3029cda4",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'citation' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn [1], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[43mcitation\u001b[49m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'citation' is not defined"
     ]
    }
   ],
   "source": [
    "citation"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c31d0a65",
   "metadata": {},
   "source": [
    "# Reproduce Uzzi et al. 2013 Figure 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3b8546f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>NoveltyScore</th>\n",
       "      <th>ConventionalityScore</th>\n",
       "      <th>Year</th>\n",
       "      <th>HitPaper</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4418464</td>\n",
       "      <td>60.252267</td>\n",
       "      <td>60.252267</td>\n",
       "      <td>1950</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7975075</td>\n",
       "      <td>7.977447</td>\n",
       "      <td>7.977447</td>\n",
       "      <td>1950</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8329296</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1950</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8591843</td>\n",
       "      <td>0.769919</td>\n",
       "      <td>0.769919</td>\n",
       "      <td>1950</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8962600</td>\n",
       "      <td>0.417585</td>\n",
       "      <td>3.897169</td>\n",
       "      <td>1950</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311959</th>\n",
       "      <td>3177134280</td>\n",
       "      <td>-10.644544</td>\n",
       "      <td>171.848535</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311960</th>\n",
       "      <td>3177259003</td>\n",
       "      <td>-56.979591</td>\n",
       "      <td>225.091840</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311961</th>\n",
       "      <td>3177284075</td>\n",
       "      <td>4.777778</td>\n",
       "      <td>39.079259</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311962</th>\n",
       "      <td>3177285925</td>\n",
       "      <td>13.250000</td>\n",
       "      <td>13.250000</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311964</th>\n",
       "      <td>3177495864</td>\n",
       "      <td>90.840037</td>\n",
       "      <td>285.657684</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11228362 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          PublicationId  NoveltyScore  ConventionalityScore  Year  HitPaper\n",
       "4               4418464     60.252267             60.252267  1950      True\n",
       "6               7975075      7.977447              7.977447  1950     False\n",
       "7               8329296      3.000000              3.000000  1950      True\n",
       "8               8591843      0.769919              0.769919  1950      True\n",
       "9               8962600      0.417585              3.897169  1950     False\n",
       "...                 ...           ...                   ...   ...       ...\n",
       "12311959     3177134280    -10.644544            171.848535  2000     False\n",
       "12311960     3177259003    -56.979591            225.091840  2000     False\n",
       "12311961     3177284075      4.777778             39.079259  2000     False\n",
       "12311962     3177285925     13.250000             13.250000  2000     False\n",
       "12311964     3177495864     90.840037            285.657684  2000     False\n",
       "\n",
       "[11228362 rows x 5 columns]"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "novelty_score = novelty_score.merge(citation[['PublicationId', 'Year', 'HitPaper']])\n",
    "novelty_score = novelty_score.dropna()\n",
    "novelty_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "372fbc3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>PublicationId</th>\n",
       "      <th>NoveltyScore</th>\n",
       "      <th>ConventionalityScore</th>\n",
       "      <th>Year</th>\n",
       "      <th>HitPaper</th>\n",
       "      <th>HighNovelty</th>\n",
       "      <th>HighConvention</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4418464</td>\n",
       "      <td>60.252267</td>\n",
       "      <td>60.252267</td>\n",
       "      <td>1950</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7975075</td>\n",
       "      <td>7.977447</td>\n",
       "      <td>7.977447</td>\n",
       "      <td>1950</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8329296</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>3.000000</td>\n",
       "      <td>1950</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>8591843</td>\n",
       "      <td>0.769919</td>\n",
       "      <td>0.769919</td>\n",
       "      <td>1950</td>\n",
       "      <td>True</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>8962600</td>\n",
       "      <td>0.417585</td>\n",
       "      <td>3.897169</td>\n",
       "      <td>1950</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311959</th>\n",
       "      <td>3177134280</td>\n",
       "      <td>-10.644544</td>\n",
       "      <td>171.848535</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311960</th>\n",
       "      <td>3177259003</td>\n",
       "      <td>-56.979591</td>\n",
       "      <td>225.091840</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311961</th>\n",
       "      <td>3177284075</td>\n",
       "      <td>4.777778</td>\n",
       "      <td>39.079259</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311962</th>\n",
       "      <td>3177285925</td>\n",
       "      <td>13.250000</td>\n",
       "      <td>13.250000</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12311964</th>\n",
       "      <td>3177495864</td>\n",
       "      <td>90.840037</td>\n",
       "      <td>285.657684</td>\n",
       "      <td>2000</td>\n",
       "      <td>False</td>\n",
       "      <td>False</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>11228362 rows × 7 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "          PublicationId  NoveltyScore  ConventionalityScore  Year  HitPaper  \\\n",
       "4               4418464     60.252267             60.252267  1950      True   \n",
       "6               7975075      7.977447              7.977447  1950     False   \n",
       "7               8329296      3.000000              3.000000  1950      True   \n",
       "8               8591843      0.769919              0.769919  1950      True   \n",
       "9               8962600      0.417585              3.897169  1950     False   \n",
       "...                 ...           ...                   ...   ...       ...   \n",
       "12311959     3177134280    -10.644544            171.848535  2000     False   \n",
       "12311960     3177259003    -56.979591            225.091840  2000     False   \n",
       "12311961     3177284075      4.777778             39.079259  2000     False   \n",
       "12311962     3177285925     13.250000             13.250000  2000     False   \n",
       "12311964     3177495864     90.840037            285.657684  2000     False   \n",
       "\n",
       "          HighNovelty  HighConvention  \n",
       "4               False           False  \n",
       "6               False           False  \n",
       "7               False           False  \n",
       "8               False           False  \n",
       "9               False           False  \n",
       "...               ...             ...  \n",
       "12311959         True            True  \n",
       "12311960         True            True  \n",
       "12311961        False           False  \n",
       "12311962        False           False  \n",
       "12311964        False            True  \n",
       "\n",
       "[11228362 rows x 7 columns]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "novelty_score['HighNovelty'] = novelty_score['NoveltyScore'] < 0\n",
    "novelty_score['HighConvention'] = novelty_score['ConventionalityScore'] > novelty_score['ConventionalityScore'].median()\n",
    "\n",
    "novelty_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "53c0de25",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "HighConvention  HighNovelty\n",
       "False           False          0.045470\n",
       "                True           0.099058\n",
       "True            False          0.090858\n",
       "                True           0.134091\n",
       "Name: HitPaper, dtype: float64"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# we get slightly higher numbers of hit papers than reported in the Uzzi paper\n",
    "# but the qualitative pattern is spot on\n",
    "novelty_score[novelty_score['Year'] >= 1990].groupby(['HighConvention', 'HighNovelty'])['HitPaper'].mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b5ab29e3",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}