2_features_extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4454, 14)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>filename</th>\n",
       "      <th>author</th>\n",
       "      <th>fulltitle</th>\n",
       "      <th>subtitle</th>\n",
       "      <th>pages</th>\n",
       "      <th>volume</th>\n",
       "      <th>number</th>\n",
       "      <th>keywords</th>\n",
       "      <th>classifications</th>\n",
       "      <th>useful</th>\n",
       "      <th>searchquery_terms</th>\n",
       "      <th>year</th>\n",
       "      <th>abstract</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Selected Characteristics of Campus-Bound Black...</td>\n",
       "      <td>Chronicle of Higher Education</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Ausschluss,Publikationsart','Quelle,Niere,eb...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>['Niere']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>; A chart is presented that lists the characte...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Temsirolimus: CCI 779, CCI-779, Cell cycle inh...</td>\n",
       "      <td>Drugs in R and D</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>['Niere']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>. The aim of this study was to determine the e...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>Bevacizumab (Avastin)</td>\n",
       "      <td>The Medical letter on drugs and therapeutics</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>['Niere']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>AE 941</td>\n",
       "      <td>Drugs in R and D</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>['Niere']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>12 months. The US FDA granted orphan drug stat...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[]</td>\n",
       "      <td>'A milestone' in treatment for renal cell cancer</td>\n",
       "      <td>European journal of cancer (Oxford, England : ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Ausschluss,NKS','Quelle,Niere,ovid,systemati...</td>\n",
       "      <td>[]</td>\n",
       "      <td>0</td>\n",
       "      <td>['Niere']</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   num filename author                                          fulltitle  \\\n",
       "0    1      NaN     []  Selected Characteristics of Campus-Bound Black...   \n",
       "1    2      NaN     []  Temsirolimus: CCI 779, CCI-779, Cell cycle inh...   \n",
       "2    3      NaN     []                              Bevacizumab (Avastin)   \n",
       "3    4      NaN     []                                             AE 941   \n",
       "4    5      NaN     []   'A milestone' in treatment for renal cell cancer   \n",
       "\n",
       "                                            subtitle  pages  volume  number  \\\n",
       "0                      Chronicle of Higher Education    NaN     NaN     NaN   \n",
       "1                                   Drugs in R and D    NaN     NaN     NaN   \n",
       "2       The Medical letter on drugs and therapeutics    NaN     NaN     NaN   \n",
       "3                                   Drugs in R and D    NaN     NaN     NaN   \n",
       "4  European journal of cancer (Oxford, England : ...    NaN     NaN     NaN   \n",
       "\n",
       "                                            keywords classifications  useful  \\\n",
       "0  ['Ausschluss,Publikationsart','Quelle,Niere,eb...              []       0   \n",
       "1  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...              []       0   \n",
       "2  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...              []       0   \n",
       "3  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...              []       0   \n",
       "4  ['Ausschluss,NKS','Quelle,Niere,ovid,systemati...              []       0   \n",
       "\n",
       "  searchquery_terms  year                                           abstract  \n",
       "0         ['Niere']   NaN  ; A chart is presented that lists the characte...  \n",
       "1         ['Niere']   NaN  . The aim of this study was to determine the e...  \n",
       "2         ['Niere']   NaN                                                NaN  \n",
       "3         ['Niere']   NaN  12 months. The US FDA granted orphan drug stat...  \n",
       "4         ['Niere']   NaN                                                NaN  "
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "# Read dataframe with extracted information\n",
    "df_niere = pd.read_csv(\"../metadata/data-dive-files/niere.csv\", escapechar='\\\\')\n",
    "\n",
    "print df_niere.shape\n",
    "df_niere.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(4334, 5)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>fulltitle</th>\n",
       "      <th>abstract</th>\n",
       "      <th>keywords</th>\n",
       "      <th>useful</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Selected Characteristics of Campus-Bound Black...</td>\n",
       "      <td>; A chart is presented that lists the characte...</td>\n",
       "      <td>['Ausschluss,Publikationsart','Quelle,Niere,eb...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Temsirolimus: CCI 779, CCI-779, Cell cycle inh...</td>\n",
       "      <td>. The aim of this study was to determine the e...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>AE 941</td>\n",
       "      <td>12 months. The US FDA granted orphan drug stat...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>5HT3-receptor antagonists as antiemetics in ca...</td>\n",
       "      <td>Effective antiemetic therapy is crucial for pa...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Comprehensive side-effect profile of anastrozo...</td>\n",
       "      <td>; p=00004). Interpretation: Anastrozole is tol...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Harnwege,ovid,syst...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   num                                          fulltitle  \\\n",
       "0    1  Selected Characteristics of Campus-Bound Black...   \n",
       "1    2  Temsirolimus: CCI 779, CCI-779, Cell cycle inh...   \n",
       "3    4                                             AE 941   \n",
       "6    7  5HT3-receptor antagonists as antiemetics in ca...   \n",
       "8    9  Comprehensive side-effect profile of anastrozo...   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  ; A chart is presented that lists the characte...   \n",
       "1  . The aim of this study was to determine the e...   \n",
       "3  12 months. The US FDA granted orphan drug stat...   \n",
       "6  Effective antiemetic therapy is crucial for pa...   \n",
       "8  ; p=00004). Interpretation: Anastrozole is tol...   \n",
       "\n",
       "                                            keywords  useful  \n",
       "0  ['Ausschluss,Publikationsart','Quelle,Niere,eb...       0  \n",
       "1  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...       0  \n",
       "3  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...       0  \n",
       "6  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...       0  \n",
       "8  ['Ausschluss,Thema','Quelle,Harnwege,ovid,syst...       0  "
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Select fulltitle and abstract of records\n",
    "df_niere_valid = df_niere.ix[(df_niere.fulltitle.notnull()) & (df_niere.abstract.notnull()),\n",
    "                            [\"num\",\"fulltitle\",\"abstract\",\"keywords\",\"useful\"]]\n",
    "\n",
    "print df_niere_valid.shape\n",
    "df_niere_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>col_0</th>\n",
       "      <th>freq</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>useful</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.850485</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.149515</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "col_0       freq\n",
       "useful          \n",
       "0       0.850485\n",
       "1       0.149515"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(df_niere_valid.useful, columns=\"freq\", normalize=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>num</th>\n",
       "      <th>fulltitle</th>\n",
       "      <th>abstract</th>\n",
       "      <th>keywords</th>\n",
       "      <th>useful</th>\n",
       "      <th>fulltitle_tokens</th>\n",
       "      <th>abstract_tokens</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Selected Characteristics of Campus-Bound Black...</td>\n",
       "      <td>; A chart is presented that lists the characte...</td>\n",
       "      <td>['Ausschluss,Publikationsart','Quelle,Niere,eb...</td>\n",
       "      <td>0</td>\n",
       "      <td>[selected, characteristics, campus, bound, bla...</td>\n",
       "      <td>[chart, presented, lists, characteristics, bla...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>Temsirolimus: CCI 779, CCI-779, Cell cycle inh...</td>\n",
       "      <td>. The aim of this study was to determine the e...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>0</td>\n",
       "      <td>[temsirolimus, cci, cci, cell, cycle, inhibitor]</td>\n",
       "      <td>[aim, study, determine, effect, temsirolimus, ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>AE 941</td>\n",
       "      <td>12 months. The US FDA granted orphan drug stat...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>0</td>\n",
       "      <td>[ae]</td>\n",
       "      <td>[months, us, fda, granted, orphan, drug, statu...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>5HT3-receptor antagonists as antiemetics in ca...</td>\n",
       "      <td>Effective antiemetic therapy is crucial for pa...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Niere,ovid,systema...</td>\n",
       "      <td>0</td>\n",
       "      <td>[ht, receptor, antagonists, antiemetics, cancer]</td>\n",
       "      <td>[effective, antiemetic, therapy, crucial, pati...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>Comprehensive side-effect profile of anastrozo...</td>\n",
       "      <td>; p=00004). Interpretation: Anastrozole is tol...</td>\n",
       "      <td>['Ausschluss,Thema','Quelle,Harnwege,ovid,syst...</td>\n",
       "      <td>0</td>\n",
       "      <td>[comprehensive, side, effect, profile, anastro...</td>\n",
       "      <td>[p, interpretation, anastrozole, tolerated, be...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   num                                          fulltitle  \\\n",
       "0    1  Selected Characteristics of Campus-Bound Black...   \n",
       "1    2  Temsirolimus: CCI 779, CCI-779, Cell cycle inh...   \n",
       "3    4                                             AE 941   \n",
       "6    7  5HT3-receptor antagonists as antiemetics in ca...   \n",
       "8    9  Comprehensive side-effect profile of anastrozo...   \n",
       "\n",
       "                                            abstract  \\\n",
       "0  ; A chart is presented that lists the characte...   \n",
       "1  . The aim of this study was to determine the e...   \n",
       "3  12 months. The US FDA granted orphan drug stat...   \n",
       "6  Effective antiemetic therapy is crucial for pa...   \n",
       "8  ; p=00004). Interpretation: Anastrozole is tol...   \n",
       "\n",
       "                                            keywords  useful  \\\n",
       "0  ['Ausschluss,Publikationsart','Quelle,Niere,eb...       0   \n",
       "1  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...       0   \n",
       "3  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...       0   \n",
       "6  ['Ausschluss,Thema','Quelle,Niere,ovid,systema...       0   \n",
       "8  ['Ausschluss,Thema','Quelle,Harnwege,ovid,syst...       0   \n",
       "\n",
       "                                    fulltitle_tokens  \\\n",
       "0  [selected, characteristics, campus, bound, bla...   \n",
       "1   [temsirolimus, cci, cci, cell, cycle, inhibitor]   \n",
       "3                                               [ae]   \n",
       "6   [ht, receptor, antagonists, antiemetics, cancer]   \n",
       "8  [comprehensive, side, effect, profile, anastro...   \n",
       "\n",
       "                                     abstract_tokens  \n",
       "0  [chart, presented, lists, characteristics, bla...  \n",
       "1  [aim, study, determine, effect, temsirolimus, ...  \n",
       "3  [months, us, fda, granted, orphan, drug, statu...  \n",
       "6  [effective, antiemetic, therapy, crucial, pati...  \n",
       "8  [p, interpretation, anastrozole, tolerated, be...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 2) From strings to numerical vectors\n",
    "# https://radimrehurek.com/gensim/tut1.html#from-strings-to-vectors\n",
    "\n",
    "import nltk\n",
    "import gensim\n",
    "\n",
    "def tokenize(text):\n",
    "    stopwords = set(nltk.corpus.stopwords.words(\"english\"))\n",
    "    return [x for x in gensim.utils.tokenize(text, lowercase=True, deacc=True, errors=\"ignore\")\n",
    "           if x not in stopwords]\n",
    "\n",
    "df_niere_valid.loc[:,\"fulltitle_tokens\"] = df_niere_valid[\"fulltitle\"].map(lambda x: tokenize(x))  \n",
    "df_niere_valid.loc[:,\"abstract_tokens\"] = df_niere_valid[\"abstract\"].map(lambda x: tokenize(x)) \n",
    "\n",
    "df_niere_valid.to_csv(\"../metadata/niere-example/df_classification_valid.csv\", index=0)\n",
    "df_niere_valid.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dictionary(3363 unique tokens: [u'limited', u'kaplan-meier', u'non-nursing', u'neurofibromatosis', u'hiv-aids']...)\n"
     ]
    }
   ],
   "source": [
    "# get dictionary from fulltitle string\n",
    "from gensim import corpora\n",
    "\n",
    "stoplist = set(nltk.corpus.stopwords.words(\"english\"))\n",
    "documents = list(df_niere_valid[\"fulltitle\"])\n",
    "\n",
    "\n",
    "# 2a) Tokenize documents: preprocess the strings by removing stopwords, lowering case.\n",
    "\n",
    "texts = [[word for word in document.lower().split() if word not in stoplist] for document in documents]\n",
    "           \n",
    "# 2b) Dictionary token-to-id: only tokens with frequency > 1 into a dictionary\n",
    "from collections import defaultdict\n",
    "frequency = defaultdict(int)\n",
    "for text in texts:\n",
    "    for token in text:\n",
    "        frequency[token] += 1\n",
    "\n",
    "texts = [[token for token in text if frequency[token] > 1]\n",
    "         for text in texts]\n",
    "\n",
    "# final dictionary\n",
    "dictionary = corpora.Dictionary(texts)\n",
    "dictionary.save('../metadata/niere-example/fulltitle.dict')\n",
    "print dictionary"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{u'limited': 42,\n",
       " u'kaplan-meier': 1442,\n",
       " u'non-nursing': 3303,\n",
       " u'neurofibromatosis': 1542,\n",
       " u'hiv-aids': 1540,\n",
       " u'dynamic': 1330,\n",
       " u'four': 570,\n",
       " u'obstruction': 3290,\n",
       " u'sleep': 357,\n",
       " u'neoplasia': 1735,\n",
       " u'constants': 3121,\n",
       " u'localized': 943,\n",
       " u'patient-reported': 678,\n",
       " u'relationships': 920,\n",
       " u'fibrofolliculomas': 2033,\n",
       " u'therapeutics': 512,\n",
       " u'medullary': 2061,\n",
       " u'physiological': 1461,\n",
       " u'containing': 3209,\n",
       " u'mediated': 3241,\n",
       " u'cm:': 957,\n",
       " u'soft-tissue': 100,\n",
       " u'ucp2': 714,\n",
       " u'magnetic': 2021,\n",
       " u'cytochrome': 2781,\n",
       " u'updated': 541,\n",
       " u'hormone': 434,\n",
       " u'risk': 173,\n",
       " u'downstream': 1051,\n",
       " u'activation': 1285,\n",
       " u'nodules': 2113,\n",
       " u\"women's\": 1571,\n",
       " u'(arry-142886)': 249,\n",
       " u'updates': 1587,\n",
       " u'suggests': 3120,\n",
       " u'cytokine:': 3208,\n",
       " u'affect': 1321,\n",
       " u'fractal': 2786,\n",
       " u'anemia': 811,\n",
       " u'ochratoxin': 1205,\n",
       " u'differentiated': 1611,\n",
       " u'pooled': 2210,\n",
       " u'progression': 538,\n",
       " u'induction': 1816,\n",
       " u'marechal': 1083,\n",
       " u'solution': 1343,\n",
       " u'000': 2874,\n",
       " u'present,': 1603,\n",
       " u'cholesterol': 634,\n",
       " u'enhance': 2593,\n",
       " u'markov': 2297,\n",
       " u'cladribine': 880,\n",
       " u'muc1': 2666,\n",
       " u'asian': 2117,\n",
       " u'concise': 1453,\n",
       " u'guidelines': 884,\n",
       " u'estimates': 3256,\n",
       " u'direct': 870,\n",
       " u'feasibility': 1663,\n",
       " u'second': 735,\n",
       " u'laparoscopic': 408,\n",
       " u'lymphoblastic': 1206,\n",
       " u'writing': 2790,\n",
       " u'periodontitis': 2884,\n",
       " u'expressive': 2789,\n",
       " u'intravenously': 1627,\n",
       " u'zeste': 1402,\n",
       " u'111in-girentuximab': 2832,\n",
       " u'excised': 2283,\n",
       " u'selected': 1,\n",
       " u'aldosteronism': 3009,\n",
       " u'bipolar': 1696,\n",
       " u'carbonic': 1502,\n",
       " u'cytokines': 1295,\n",
       " u'reconstruction': 759,\n",
       " u'chemoimmunotherapy': 2519,\n",
       " u'new': 58,\n",
       " u'net': 2910,\n",
       " u'photodynamic': 589,\n",
       " u'coefficient': 1392,\n",
       " u'aua': 1724,\n",
       " u'evolving': 1293,\n",
       " u'behavior': 1656,\n",
       " u'oocyte': 3271,\n",
       " u'men': 1014,\n",
       " u'(inlyta)': 3207,\n",
       " u'neurotoxicity': 2812,\n",
       " u'reported': 1787,\n",
       " u'protection': 1038,\n",
       " u'modelling': 2705,\n",
       " u'active': 951,\n",
       " u'path': 595,\n",
       " u'pancreas': 3107,\n",
       " u'obtained': 1577,\n",
       " u'balance': 2218,\n",
       " u'apoptosis,': 2686,\n",
       " u'study': 38,\n",
       " u'economics': 2877,\n",
       " u'proteinuria:': 2370,\n",
       " u'regulators': 1877,\n",
       " u'reports': 1774,\n",
       " u'pooling': 1969,\n",
       " u'granulomatosis': 2075,\n",
       " u'p,': 3185,\n",
       " u'smoke': 1966,\n",
       " u'changes': 487,\n",
       " u'decision-making': 3058,\n",
       " u'interferon': 1313,\n",
       " u'psoriasis:': 3082,\n",
       " u'classification': 1868,\n",
       " u'2009:': 2049,\n",
       " u'population:': 400,\n",
       " u'highly': 45,\n",
       " u'psychological': 48,\n",
       " u'chemotherapy,': 3115,\n",
       " u'jm,': 1080,\n",
       " u'lung,': 1641,\n",
       " u'total': 1942,\n",
       " u'landscape': 1150,\n",
       " u'genomics': 2163,\n",
       " u'dna': 375,\n",
       " u'm,': 3180,\n",
       " u'hospital': 1020,\n",
       " u'achieve': 598,\n",
       " u'investigational': 1590,\n",
       " u't-cell': 2115,\n",
       " u\"crohn's\": 2897,\n",
       " u'assessment': 41,\n",
       " u'overlooked': 2592,\n",
       " u'preparation': 2911,\n",
       " u'angiogenesis,': 1871,\n",
       " u'offline:': 2278,\n",
       " u'populations': 1841,\n",
       " u'successful': 808,\n",
       " u'(3-month)': 3078,\n",
       " u'infusion': 2394,\n",
       " u'warm': 193,\n",
       " u'adult': 1272,\n",
       " u'(fabps)': 3163,\n",
       " u'real-time': 1517,\n",
       " u'hold': 2826,\n",
       " u'pouch': 2900,\n",
       " u'harnessing': 2727,\n",
       " u'antiangiogenic': 413,\n",
       " u'mg': 1849,\n",
       " u'lapatinib': 1229,\n",
       " u'intronic': 1404,\n",
       " u'mammalian': 464,\n",
       " u'antigen-4': 1917,\n",
       " u'receptor-2': 2080,\n",
       " u'era': 965,\n",
       " u'interaction': 2181,\n",
       " u'tnm': 973,\n",
       " u'ta': 1989,\n",
       " u'appraisal': 617,\n",
       " u'example': 1225,\n",
       " u'unilateral': 3005,\n",
       " u'ramucirumab': 615,\n",
       " u'household': 1574,\n",
       " u'therapy:': 889,\n",
       " u'therapy?': 1068,\n",
       " u'tivozanib': 730,\n",
       " u'dosing': 1028,\n",
       " u'non-conventional': 2566,\n",
       " u'fludarabine': 2742,\n",
       " u'reviewing': 2916,\n",
       " u'angiotensin-converting': 2621,\n",
       " u'counseling': 3174,\n",
       " u'validated': 1437,\n",
       " u'therapy,': 133,\n",
       " u'advancing': 625,\n",
       " u'absolute': 1167,\n",
       " u'endometrial': 1032,\n",
       " u'end': 966,\n",
       " u'predictors': 1469,\n",
       " u'colonic': 657,\n",
       " u'sitting': 3175,\n",
       " u'damage': 708,\n",
       " u'co-morbidities': 2127,\n",
       " u'lesions': 418,\n",
       " u'yttrium-90': 2567,\n",
       " u'leiomyoma': 326,\n",
       " u'significance': 367,\n",
       " u'perioperative': 510,\n",
       " u'symposium': 91,\n",
       " u'aurora': 2823,\n",
       " u'translocation': 2976,\n",
       " u'1q43': 321,\n",
       " u'radiosurgery': 1139,\n",
       " u'undergoing': 189,\n",
       " u'modulation': 869,\n",
       " u'arthritis': 584,\n",
       " u'radium-223': 1369,\n",
       " u'therapies:': 908,\n",
       " u'calcitriol': 2554,\n",
       " u'low-stage': 1096,\n",
       " u'inactivity': 1171,\n",
       " u'effective': 2055,\n",
       " u'ablation': 311,\n",
       " u'orally': 1203,\n",
       " u'algorithms': 1894,\n",
       " u'iii,': 3108,\n",
       " u'correlation': 859,\n",
       " u'rehospitalisation': 430,\n",
       " u'preventive': 1076,\n",
       " u'axitinib:': 3289,\n",
       " u'wine': 3284,\n",
       " u'cumulative': 2141,\n",
       " u'interpretation': 2719,\n",
       " u'lenvatinib,': 2819,\n",
       " u'disease?': 977,\n",
       " u'disease:': 257,\n",
       " u'double-blinded,': 2858,\n",
       " u'ablation:': 320,\n",
       " u'innovative': 2329,\n",
       " u'japan': 1764,\n",
       " u'disease,': 2044,\n",
       " u'first-line': 1244,\n",
       " u'genome-wide': 2006,\n",
       " u'ligand': 1729,\n",
       " u'dermatologic': 1364,\n",
       " u'personal': 922,\n",
       " u'multidisciplinary': 77,\n",
       " u'folate': 2429,\n",
       " u'better': 22,\n",
       " u'end-point': 2845,\n",
       " u'differently': 1430,\n",
       " u'acupuncture': 2503,\n",
       " u'non-small': 1642,\n",
       " u'strength': 3224,\n",
       " u'overcome': 3050,\n",
       " u'versus': 780,\n",
       " u'ecog-acrin': 1898,\n",
       " u'coffee': 619,\n",
       " u'diseases': 771,\n",
       " u'combination': 227,\n",
       " u'atherosclerosis': 2534,\n",
       " u's-transferase': 210,\n",
       " u'therapy': 36,\n",
       " u'effects': 119,\n",
       " u'one': 2171,\n",
       " u'structural': 2170,\n",
       " u\"administration's\": 2399,\n",
       " u'bony': 3301,\n",
       " u'meat': 483,\n",
       " u'repurposing': 934,\n",
       " u'transarterial': 1130,\n",
       " u'nonsmall': 3069,\n",
       " u'ventilation': 1679,\n",
       " u'hiv-infected': 219,\n",
       " u'fitness': 1753,\n",
       " u'nephron-sparing': 1081,\n",
       " u'bone': 146,\n",
       " u'mean': 972,\n",
       " u'cpg': 3210,\n",
       " u'series': 1550,\n",
       " u'spanish': 2748,\n",
       " u'principles': 2471,\n",
       " u'assure,': 2159,\n",
       " u'neuroleptic': 2379,\n",
       " u'used': 1785,\n",
       " u'oncological': 868,\n",
       " u'combining': 599,\n",
       " u'aids': 910,\n",
       " u'extract': 1233,\n",
       " u'blinded': 537,\n",
       " u'fatty': 2882,\n",
       " u'thrombus': 2584,\n",
       " u'ym155': 3153,\n",
       " u'network': 185,\n",
       " u'integrative': 2735,\n",
       " u'particles': 1372,\n",
       " u'combinatorial': 3134,\n",
       " u'dysplastic': 385,\n",
       " u'medicine': 991,\n",
       " u'newly': 2842,\n",
       " u'acrylamide': 2249,\n",
       " u'(construct)': 3073,\n",
       " u'temporal': 1738,\n",
       " u'healing:': 844,\n",
       " u'u.s.': 395,\n",
       " u'free': 2114,\n",
       " u'standard': 1327,\n",
       " u'hematologic': 1946,\n",
       " u'amadeus': 608,\n",
       " u'chromosomal': 831,\n",
       " u'mononuclear': 1219,\n",
       " u'post-approval': 1895,\n",
       " u'non-prescribed': 2294,\n",
       " u'pharmacological': 3049,\n",
       " u'titration:': 3019,\n",
       " u'generic,': 1433,\n",
       " u'diabetic': 2219,\n",
       " u'descriptive': 2573,\n",
       " u'melanoma-derived': 2987,\n",
       " u'organ-specific': 2360,\n",
       " u'smokers': 1241,\n",
       " u'angiogenic': 3056,\n",
       " u'r,': 1091,\n",
       " u'garlic': 2401,\n",
       " u'hippel-lindau': 1366,\n",
       " u'hand-foot': 894,\n",
       " u'preoperative': 377,\n",
       " u'features': 411,\n",
       " u'grade': 2888,\n",
       " u'proteomics': 2282,\n",
       " u'potentials': 2647,\n",
       " u'primary': 127,\n",
       " u'nanotechnology-based': 2850,\n",
       " u'production': 2964,\n",
       " u'another': 2671,\n",
       " u'robust': 2933,\n",
       " u'oncogenic': 299,\n",
       " u'antibiotic': 3139,\n",
       " u'dysfunction': 1209,\n",
       " u'seizures': 3138,\n",
       " u'open-label': 2343,\n",
       " u'cisplatin-based': 1597,\n",
       " u'opportunity': 2930,\n",
       " u'approach?': 2351,\n",
       " u'approach:': 2253,\n",
       " u'needed': 39,\n",
       " u'rates': 167,\n",
       " u'listed': 1409,\n",
       " u'analgesic': 2204,\n",
       " u'analgesia': 845,\n",
       " u'legs': 2377,\n",
       " u'exposures': 1363,\n",
       " u'smoking-related': 3246,\n",
       " u'antihypertensive': 2620,\n",
       " u'enhancing': 1450,\n",
       " u'arising': 1467,\n",
       " u'tool': 428,\n",
       " u'angiomyolipoma': 995,\n",
       " u'promise': 2372,\n",
       " u'japanese': 1943,\n",
       " u'treatment-related': 2268,\n",
       " u'mva-5t4': 2194,\n",
       " u'peculiar': 1568,\n",
       " u'symptom': 926,\n",
       " u'p53': 1053,\n",
       " u'exposure,': 3018,\n",
       " u'nail': 2035,\n",
       " u'biopsy:': 571,\n",
       " u\"down's\": 1719,\n",
       " u'target': 463,\n",
       " u'roles': 1294,\n",
       " u'promise?': 2827,\n",
       " u'older': 501,\n",
       " u'zero': 2312,\n",
       " u'regulator': 298,\n",
       " u'pneumonia': 795,\n",
       " u'leg:': 2077,\n",
       " u'absence': 1795,\n",
       " u'stratified': 2427,\n",
       " u'iron': 695,\n",
       " u'nivolumab': 295,\n",
       " u'q192r': 3215,\n",
       " u'factor-634': 790,\n",
       " u'raf': 1897,\n",
       " u'tomography/computed': 2866,\n",
       " u'endocrinology': 1813,\n",
       " u'modern': 2346,\n",
       " u'spectrum': 1011,\n",
       " u'attenuation': 2338,\n",
       " u'transcatheter': 2660,\n",
       " u'(rapalogs)': 3052,\n",
       " u'benefit-risk': 947,\n",
       " u'compression': 2500,\n",
       " u'intratumoural': 317,\n",
       " u'juxtaglomerular': 2440,\n",
       " u'medline': 2576,\n",
       " u'cardiac': 1063,\n",
       " u'relapse': 1747,\n",
       " u'gstt1': 3292,\n",
       " u'pleural': 932,\n",
       " u'ki-67': 2581,\n",
       " u'latter': 3030,\n",
       " u'-': 54,\n",
       " u'isolated': 436,\n",
       " u'chest': 113,\n",
       " u'subjects': 286,\n",
       " u'cohort,': 2097,\n",
       " u'placebo-controlled': 574,\n",
       " u'anti-interleukin-6': 2972,\n",
       " u'effectiveness': 1425,\n",
       " u'intramyopericardial': 3033,\n",
       " u'uro-oncology:': 2383,\n",
       " u'dermal': 2846,\n",
       " u'single-nucleotide': 339,\n",
       " u'toxic': 2791,\n",
       " u'methylphenidate': 2857,\n",
       " u'immunogenicity': 447,\n",
       " u'metrics': 2418,\n",
       " u'involving': 2958,\n",
       " u'carcinoma': 61,\n",
       " u'vaccinia': 525,\n",
       " u'quantitative': 481,\n",
       " u'phase': 234,\n",
       " u'assessed': 361,\n",
       " u'papillomavirus': 940,\n",
       " u'autopsy': 1653,\n",
       " u'immunological': 1701,\n",
       " u'saxagliptin:': 2246,\n",
       " u'intergroup': 268,\n",
       " u'meta-analyses': 1353,\n",
       " u'medical': 639,\n",
       " u'urogenital': 1801,\n",
       " u'pazopanib,': 2175,\n",
       " u'neuroleptic-induced': 2375,\n",
       " u'points': 2927,\n",
       " u'anticancer': 596,\n",
       " u'quantifying': 1301,\n",
       " u'silencing': 2691,\n",
       " u'high-intensity': 3024,\n",
       " u'breastfeeding': 2563,\n",
       " u'd,': 3191,\n",
       " u'signatures': 1384,\n",
       " u'reductions': 1804,\n",
       " u'brief': 1008,\n",
       " u'soluble': 2415,\n",
       " u'd:': 629,\n",
       " u'protocol': 1939,\n",
       " u'papillary': 1509,\n",
       " u'home-acquired': 3305,\n",
       " u'lessons': 1749,\n",
       " u'laparoendoscopic': 699,\n",
       " u'guidance': 2746,\n",
       " u'kinase': 63,\n",
       " u'pten': 2940,\n",
       " u'screening': 547,\n",
       " u'chromophobe': 2785,\n",
       " u'hutson': 3203,\n",
       " u'palliative': 839,\n",
       " u'epidermal': 1281,\n",
       " u'overactivated': 3132,\n",
       " u'mixture': 1607,\n",
       " u'dihydrochloride': 2751,\n",
       " u'high-dose': 668,\n",
       " u'umbilical': 1478,\n",
       " u'da': 1085,\n",
       " u'lebanon:': 2482,\n",
       " u'meeting,': 73,\n",
       " u'vaccinations': 2135,\n",
       " u'despite': 1226,\n",
       " u'report': 656,\n",
       " u'metallothionein': 2140,\n",
       " u'nurse': 3230,\n",
       " u'interleukin-2-': 2520,\n",
       " u'interleukin-2,': 3338,\n",
       " u'contributes': 1277,\n",
       " u'enoxaparin': 1217,\n",
       " u'men:': 1781,\n",
       " u'countries': 2106,\n",
       " u'clinicopathologic': 259,\n",
       " u'ureteroscopy': 2366,\n",
       " u'unmet': 2173,\n",
       " u'shape': 2425,\n",
       " u'twice': 1227,\n",
       " u'three-year': 1589,\n",
       " u'release': 1017,\n",
       " u'fertility': 2923,\n",
       " u'wt1': 1617,\n",
       " u'sparing': 1533,\n",
       " u'peripheral': 1220,\n",
       " u'testing': 1444,\n",
       " u'non-steroidal': 3093,\n",
       " u'alterations': 261,\n",
       " u'detection': 1513,\n",
       " u'adult-strength': 2373,\n",
       " u'depends': 2640,\n",
       " u'result': 1991,\n",
       " u'sporadic': 1000,\n",
       " u'depressive': 2381,\n",
       " u'alternative': 1796,\n",
       " u'best': 1968,\n",
       " u'techniques': 409,\n",
       " u'capacity': 470,\n",
       " u'connective': 2801,\n",
       " u'matter?': 3067,\n",
       " u'score': 765,\n",
       " u'psychological,': 2797,\n",
       " u'seven': 2335,\n",
       " u'dialysis:': 1307,\n",
       " u'occupational': 1361,\n",
       " u'approach': 130,\n",
       " u'discovery': 1892,\n",
       " u'preserve': 2879,\n",
       " u'epidemiological': 2145,\n",
       " u'positron': 1337,\n",
       " u'lymphocyte': 978,\n",
       " u'vitamin': 102,\n",
       " u'wt': 2970,\n",
       " u'386': 3011,\n",
       " u'clues': 1905,\n",
       " u'patients?': 57,\n",
       " u'modeling': 1057,\n",
       " u'patients:': 169,\n",
       " u'improve': 1179,\n",
       " u'patented': 198,\n",
       " u'agonist': 1625,\n",
       " u'resistant': 3070,\n",
       " u'abstracts': 84,\n",
       " u'ill': 2509,\n",
       " u'ventricular': 355,\n",
       " u'allopurinol': 2426,\n",
       " u'und': 111,\n",
       " u'anatomic': 1935,\n",
       " u'china': 2715,\n",
       " u'contribution': 2807,\n",
       " u'hypertension:': 1476,\n",
       " u'collecting': 3330,\n",
       " u'intake,': 2201,\n",
       " u'vein': 2734,\n",
       " u'radiopharmaceuticals': 1370,\n",
       " u'anti-angiogenesis': 1797,\n",
       " u'mutations': 2108,\n",
       " u'height': 2098,\n",
       " u'cholelithiasis': 3156,\n",
       " u'viruses': 840,\n",
       " u'applicability': 1777,\n",
       " u'pregnancy': 2922,\n",
       " u'arsenic': 3035,\n",
       " u'pre-selected': 3261,\n",
       " u'diagnosed': 2803,\n",
       " u'strong': 3225,\n",
       " u'targeting': 256,\n",
       " u'basis': 2931,\n",
       " u'chrome': 1549,\n",
       " u'three': 1944,\n",
       " u'extraction': 984,\n",
       " u'castration-resistant': 1440,\n",
       " u'now?': 627,\n",
       " u'chemoprevention:': 2830,\n",
       " u'interest': 2248,\n",
       " u'flexible': 688,\n",
       " u'angiomyolipoma:': 2687,\n",
       " u'rs895819a/g': 3280,\n",
       " u'obese': 3110,\n",
       " u'life': 739,\n",
       " u'families': 1632,\n",
       " u'efficacy': 44,\n",
       " u'eastern': 2092,\n",
       " u'drugs': 583,\n",
       " u'bioavailability': 1166,\n",
       " u\"d'urologie\": 96,\n",
       " u'relevance': 1230,\n",
       " u'anticoagulation:': 607,\n",
       " u'immunotherapy:': 457,\n",
       " u'(aldh9a1)': 2229,\n",
       " u'cytotoxic': 852,\n",
       " u'immunotherapies': 1291,\n",
       " u'harms': 1013,\n",
       " u'telephone-based': 500,\n",
       " u'japan:': 3286,\n",
       " u'antiretroviral': 32,\n",
       " u'metaanalysis': 3295,\n",
       " u'calcium': 1459,\n",
       " u'angiomyolipomas': 2449,\n",
       " u'coffin': 2050,\n",
       " u'endpoints': 218,\n",
       " u'tumor-associated': 522,\n",
       " u'characterizing': 2651,\n",
       " u'(evaluating': 611,\n",
       " u'ix': 1498,\n",
       " u'azathioprine': 1537,\n",
       " u'concomitant': 443,\n",
       " u'membranous': 1748,\n",
       " u'glutamic': 2811,\n",
       " u'ureter': 2037,\n",
       " u'ii': 272,\n",
       " u'tissue': 312,\n",
       " u'post-transplant': 1817,\n",
       " u'abnormalities': 1758,\n",
       " u'comparative': 1356,\n",
       " u'tumorigenesis': 1276,\n",
       " u'pathology': 497,\n",
       " u'mouse': 1416,\n",
       " u'jp,': 3178,\n",
       " u'percutaneous': 319,\n",
       " u'recurrent': 229,\n",
       " u'wound': 842,\n",
       " u'program:': 2873,\n",
       " u'fatal': 1860,\n",
       " u'complex': 998,\n",
       " u'potentially': 548,\n",
       " u'phenotype': 1456,\n",
       " u'30986': 1659,\n",
       " u'practice:': 2699,\n",
       " u'lithium': 2475,\n",
       " u'rna': 1565,\n",
       " u'european': 82,\n",
       " u'prevalence,': 3220,\n",
       " u'independent': 535,\n",
       " u'cancer-analysis': 2390,\n",
       " u'social': 2279,\n",
       " u'malignancy:': 2560,\n",
       " u'hand': 280,\n",
       " u'action': 1374,\n",
       " u'portal': 1477,\n",
       " u'i,': 1089,\n",
       " u'immunohistochemical': 2138,\n",
       " u'overt': 2158,\n",
       " u'cost-effectiveness': 532,\n",
       " u'cycle': 5,\n",
       " u'allografts': 3133,\n",
       " u'scenario': 954,\n",
       " u'adolescents': 2551,\n",
       " u'disseminated': 1925,\n",
       " u'studer': 3103,\n",
       " u'warburg': 710,\n",
       " u'squamous': 1067,\n",
       " u'hydration': 117,\n",
       " u'malat-1': 3148,\n",
       " u'studying': 2130,\n",
       " u'claims': 1243,\n",
       " u'paradigm': 941,\n",
       " u'left': 362,\n",
       " u'agency': 3206,\n",
       " u'cisplatin': 782,\n",
       " u'nephrectomy?': 3146,\n",
       " u'nephrectomy:': 407,\n",
       " u'peptide,': 2070,\n",
       " u'identify': 1447,\n",
       " u'gustave': 3197,\n",
       " u'human': 529,\n",
       " u'facts': 810,\n",
       " u'article': 3136,\n",
       " u'modulate': 1288,\n",
       " u'previous': 1258,\n",
       " u'thyroid': 391,\n",
       " u'adding': 3172,\n",
       " u'(pd-l1)': 3275,\n",
       " u'atrial': 282,\n",
       " u'candidate': 1352,\n",
       " u'stent': 2828,\n",
       " u'complex:': 1600,\n",
       " u'pharmacy': 1604,\n",
       " u'topical': 2034,\n",
       " u'neoplasm': 736,\n",
       " u'advanced': 154,\n",
       " u'detection:': 2713,\n",
       " u'plasma': 1823,\n",
       " u'elevation': 160,\n",
       " u'one-carbon': 2736,\n",
       " u'advances': 435,\n",
       " u'survival': 477,\n",
       " u'progressed': 850,\n",
       " u'possible': 1890,\n",
       " u'saracatinib': 821,\n",
       " u'fusion': 580,\n",
       " u'vdr': 2955,\n",
       " u'lymphoma': 1928,\n",
       " u'therapy-brm': 742,\n",
       " u'evidence?': 905,\n",
       " u'atria,': 601,\n",
       " u'pharmacist': 3231,\n",
       " u'nude': 1414,\n",
       " u'pharmacodynamics': 2730,\n",
       " u'specific': 1593,\n",
       " u'occlusions': 3211,\n",
       " u'immunosuppressive': 2063,\n",
       " u'hepatic': 181,\n",
       " u'analyzing': 2590,\n",
       " u'skeletal-related': 1123,\n",
       " u'il-2-based': 2128,\n",
       " u'(avoren):': 1830,\n",
       " u'vaccae': 2912,\n",
       " u'beneficial': 2688,\n",
       " u'right': 552,\n",
       " u'old': 1129,\n",
       " u'protein-1': 1955,\n",
       " u'korea': 2501,\n",
       " u'people': 2245,\n",
       " u'(fgfr)': 2752,\n",
       " u'elderly': 398,\n",
       " u'negrier': 3199,\n",
       " u'pathologies:': 3212,\n",
       " u'histopathologic': 380,\n",
       " u'9p': 1789,\n",
       " u'critique': 2594,\n",
       " u'pain': 744,\n",
       " u'indirect': 2633,\n",
       " u'short-term': 1421,\n",
       " u'evidence,': 2365,\n",
       " u'non-inferiority': 1215,\n",
       " u'microwave': 3300,\n",
       " u'aberrant': 2792,\n",
       " u'unit': 1961,\n",
       " u'anti-pd-1': 3140,\n",
       " u'intolerant': 2065,\n",
       " u'pazopanib-induced': 2238,\n",
       " u'cortical': 1466,\n",
       " u'collaboration': 1722,\n",
       " u'cord': 1263,\n",
       " u'core': 3130,\n",
       " u'r337h': 1108,\n",
       " u'canagliflozin': 2909,\n",
       " u'predicting': 1350,\n",
       " u'six2': 3112,\n",
       " u'caffeine': 2199,\n",
       " u'metastatic': 40,\n",
       " u'cancer-related': 857,\n",
       " u'post': 480,\n",
       " u'telomere': 2256,\n",
       " u'super': 2967,\n",
       " u'carcinomas:': 686,\n",
       " u'heparin': 2068,\n",
       " u'antiepileptics': 3296,\n",
       " u't1a': 1112,\n",
       " u't1b': 1113,\n",
       " u'rheumatoid': 2045,\n",
       " u'ileal': 3100,\n",
       " u'months': 658,\n",
       " u'adipose': 2125,\n",
       " u'prevention:': 2094,\n",
       " u'costs': 2186,\n",
       " u'cell-based': 1973,\n",
       " u'nephroma:': 1810,\n",
       " u'plus': 274,\n",
       " u'done': 2423,\n",
       " u'status,': 1692,\n",
       " u'prevention,': 2732,\n",
       " u'has-bled': 600,\n",
       " u'considerations': 236,\n",
       " u'raised': 540,\n",
       " u'nodules:': 3260,\n",
       " u'mantle': 47,\n",
       " u'wasting': 64,\n",
       " u'methotrexate/carboplatin/vinblastine': 1660,\n",
       " u'tract': 371,\n",
       " u'interferonalpha': 3189,\n",
       " u'carcinoid):': 3259,\n",
       " u'usefulness': 2535,\n",
       " u'anti-cd137': 3297,\n",
       " u'soy': 2852,\n",
       " u'reducing': 833,\n",
       " u'(cyp1a1)': 2778,\n",
       " u'malignancies': 389,\n",
       " u'dehydrogenase': 635,\n",
       " u'novo': 645,\n",
       " u'trial):': 1648,\n",
       " u'surgery:': 803,\n",
       " u'vaccination': 919,\n",
       " u'cancer.': 1570,\n",
       " u'support': 1145,\n",
       " u'initial': 331,\n",
       " u'available': 1744,\n",
       " u'erythrocyte': 161,\n",
       " u'mutated': 2653,\n",
       " u'clinical': 35,\n",
       " u'way': 871,\n",
       " u'complications,': 3362,\n",
       " u'cancer?': 2564,\n",
       " u'metabolism': 988,\n",
       " u'synthesis': 1164,\n",
       " u'intolerance': 3048,\n",
       " u'head': 929,\n",
       " u'metronomic': 2039,\n",
       " u'ckd': 2750,\n",
       " u'clinics': 3285,\n",
       " u'becoming': 1768,\n",
       " u'differences': 2702,\n",
       " u'controlled,': 1836,\n",
       " u'failure': 393,\n",
       " u'heat': 137,\n",
       " u'prediagnostic': 3223,\n",
       " u'solar': 2086,\n",
       " u'rituximab': 387,\n",
       " u'fibrillation)': 609,\n",
       " u'approval': 2047,\n",
       " u'docetaxel': 781,\n",
       " u'nationwide': 1448,\n",
       " u'survival,': 1675,\n",
       " u'(fact-brm):': 743,\n",
       " u'isoflavones:': 2848,\n",
       " u'type': 495,\n",
       " u'maximum': 1331,\n",
       " u'vivo': 651,\n",
       " u'complications': 213,\n",
       " u'survival:': 861,\n",
       " u'emotional': 2693,\n",
       " u'stages': 277,\n",
       " u'optimisation': 2886,\n",
       " u'standardised,': 1439,\n",
       " u'pricing': 2537,\n",
       " u'medication': 2514,\n",
       " u'hemorrhage': 2188,\n",
       " u'infections': 2726,\n",
       " u'224ra': 3249,\n",
       " u'agents:': 203,\n",
       " u'muscular': 3226,\n",
       " u'evidence': 620,\n",
       " u'zoledronate': 2029,\n",
       " u'gland': 731,\n",
       " u'simulations': 1536,\n",
       " u'creatinine': 1025,\n",
       " u'k': 946,\n",
       " u'physical': 873,\n",
       " u'cystic': 733,\n",
       " u'melanoma': 555,\n",
       " u'dying': 1736,\n",
       " u'setting': 1006,\n",
       " u'role': 622,\n",
       " u'digital': 3237,\n",
       " u'test': 2441,\n",
       " u'prospective,': 847,\n",
       " u'node': 901,\n",
       " u'factors,': 717,\n",
       " u'abscopal': 2758,\n",
       " u'models': 1963,\n",
       " u'update': 303,\n",
       " u'diet': 1100,\n",
       " u'outpatient': 1960,\n",
       " u'distribution?': 2481,\n",
       " u'self-management': 1762,\n",
       " u'factors:': 2243,\n",
       " u'and/or': 1923,\n",
       " u'glucocorticoids': 2470,\n",
       " u'scores': 604,\n",
       " u'replacement': 612,\n",
       " u'procedures,': 2847,\n",
       " u'n,': 3177,\n",
       " u'longer': 3081,\n",
       " u'mitogen-activated': 247,\n",
       " u'microemulsion': 2822,\n",
       " u'elderly:': 427,\n",
       " u'interval': 323,\n",
       " u'mitoxantrone': 1847,\n",
       " u'daily': 1228,\n",
       " u'anti-vegf': 1283,\n",
       " u'reversible': 1378,\n",
       " u'workers:': 2026,\n",
       " u'time': 836,\n",
       " u'chemoembolization': 1131,\n",
       " u'angiogenesis': 763,\n",
       " u'all-cause': 2349,\n",
       " u'urinary': 685,\n",
       " u'twice-daily': 3038,\n",
       " u'concept': 806,\n",
       " u'managed': 499,\n",
       " u'pd-1/pd-l1': 1975,\n",
       " u'chain': 1516,\n",
       " u'global': 1499,\n",
       " u'alternatives': 2190,\n",
       " u'focus': 149,\n",
       " u'integration': 2707,\n",
       " u'leads': 1445,\n",
       " u'colorectal,': 1843,\n",
       " u'physicochemical': 3315,\n",
       " u'asymptomatic': 1759,\n",
       " u'calgb': 3015,\n",
       " u'skin': 473,\n",
       " u'adenocarcinoma:': 2824,\n",
       " u'inverse': 475,\n",
       " u'impairment:': 2362,\n",
       " u'disorders': 2054,\n",
       " u'malat1': 3243,\n",
       " u'paraoxonase': 1396,\n",
       " u'technique': 1532,\n",
       " u'parenchyma': 3021,\n",
       " u'recurrence': 716,\n",
       " u'characterisation': 2960,\n",
       " u'promoting': 711,\n",
       " u'denosumab': 1200,\n",
       " u'monocytes': 1457,\n",
       " u'veterans': 2652,\n",
       " u'multitargeted': 799,\n",
       " u'ifosfamide': 1525,\n",
       " u'protective': 826,\n",
       " u'advantage': 3122,\n",
       " u'biodistribution': 392,\n",
       " u'orthotopic': 1266,\n",
       " u'nephropathy': 1745,\n",
       " u'cancer-testis': 2988,\n",
       " u'lynch': 918,\n",
       " u'10-year': 2669,\n",
       " u'nerve': 2205,\n",
       " u'lymphadenectomy': 2538,\n",
       " u'minute': 1711,\n",
       " u'responders': 2966,\n",
       " u'prophylactic': 2162,\n",
       " u'level': 679,\n",
       " u'mycophenolate': 646,\n",
       " u'taiwan:': 2397,\n",
       " u'added': 2809,\n",
       " u'surgically': 2284,\n",
       " u'caki-1': 1884,\n",
       " u'upper': 364,\n",
       " u'unexpected': 498,\n",
       " u'postsurgical': 841,\n",
       " u'immunomodulating': 2523,\n",
       " u'findings': 258,\n",
       " u'trend': 2863,\n",
       " u'group,': 3361,\n",
       " u'prediction': 378,\n",
       " u'moderate-to-severe': 779,\n",
       " u'medroxyprogesterone': 2603,\n",
       " u'hospitalised': 429,\n",
       " u'agonists': 2261,\n",
       " u'targets': 243,\n",
       " u'psychosocial': 1807,\n",
       " u'sinus:': 3032,\n",
       " u'canagliflozin,': 2589,\n",
       " u'health:': 3124,\n",
       " u'current': 305,\n",
       " u'sequential': 1875,\n",
       " u'international': 131,\n",
       " u'leukemia:': 1208,\n",
       " u'safety,': 820,\n",
       " u'paraneoplastic': 1906,\n",
       " u'ifnalpha': 3344,\n",
       " u'hairy': 878,\n",
       " u'clinical-based': 1481,\n",
       " u'agonist,': 2433,\n",
       " u'metabolomic': 2409,\n",
       " u'genes': 208,\n",
       " u'metastasizing': 384,\n",
       " u'french': 1492,\n",
       " u'understanding': 2472,\n",
       " u'water': 1458,\n",
       " u'target?': 465,\n",
       " u'holistic': 2325,\n",
       " u'groups': 2236,\n",
       " u'alone': 1213,\n",
       " u'cross-over': 1838,\n",
       " u'change': 336,\n",
       " u'cranial': 2542,\n",
       " u'noncoding': 1857,\n",
       " u'canadian': 1298,\n",
       " u'healthy': 445,\n",
       " u'salvage': 2352,\n",
       " u'finnish': 2250,\n",
       " u'somatic': 2569,\n",
       " u'trials:': 534,\n",
       " u'trial': 15,\n",
       " u'risk-prediction': 606,\n",
       " u'simultaneous': 2546,\n",
       " u'once-daily': 1829,\n",
       " u'pediatric': 233,\n",
       " u'cyst': 1465,\n",
       " u'biopsies': 2915,\n",
       " u'history': 372,\n",
       " u'studies': 769,\n",
       " u'montreal': 2981,\n",
       " u'single-port': 2447,\n",
       " u'rheumatic': 916,\n",
       " u'useful': 2683,\n",
       " u'beer,': 3283,\n",
       " u'rash': 2059,\n",
       " u'chemotherapy-related': 694,\n",
       " u'marker': 1107,\n",
       " u'antagonists': 8,\n",
       " u'relating': 1535,\n",
       " u'market': 2749,\n",
       " u'transplant': 441,\n",
       " u'allogeneic': 660,\n",
       " u'alpha-2a': 2998,\n",
       " u'survivors:': 2760,\n",
       " u'urology': 1007,\n",
       " u'working': 124,\n",
       " u'positive': 353,\n",
       " u'anatomical': 2408,\n",
       " u'ramucirumab:': 2079,\n",
       " u'gynecologic': 1840,\n",
       " u'criteria': 1921,\n",
       " u'review:': 2105,\n",
       " u'australian': 2166,\n",
       " u'ischemia': 196,\n",
       " u'today': 2445,\n",
       " u'ischemic': 1149,\n",
       " u'apparent': 1393,\n",
       " u'symptomatic': 1218,\n",
       " u'validity': 2559,\n",
       " u'antigens': 718,\n",
       " u'non-gastrointestinal': 3092,\n",
       " u'cases': 279,\n",
       " u'intravesical': 1988,\n",
       " u'genipin': 715,\n",
       " u'states': 1674,\n",
       " u'tp53': 1109,\n",
       " u'hydrochloride': 2462,\n",
       " u'modified': 1015,\n",
       " u'enucleoresection': 2892,\n",
       " u'cutaneous': 2270,\n",
       " u'anemia?': 2512,\n",
       " u'vegf': 342,\n",
       " u'mortality': 432,\n",
       " u'anemia:': 696,\n",
       " u'following': 775,\n",
       " u'making': 1347,\n",
       " u'dosage': 1859,\n",
       " u'deceased': 2438,\n",
       " u'heart': 1412,\n",
       " u'nursing': 1134,\n",
       " u'predict': 1605,\n",
       " u'expanding': 1110,\n",
       " u'alendronate': 1197,\n",
       " u'catheters': 1485,\n",
       " u'intake': 2102,\n",
       " ...}"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# checking tokens\n",
    "dictionary.token2id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# 2c) Convert tokenized documents from (a) to vector, using dictionary in (b) into a corpus\n",
    "# tokens not found on dict will be ignored\n",
    "corpus = [dictionary.doc2bow(text) for text in texts]\n",
    "corpora.MmCorpus.serialize('../metadata/niere-example/fulltitle.mm', corpus)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}