Milk_Abstract_Extraction.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import importlib\n",
    "import seaborn as sns\n",
    "from matplotlib import pyplot\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "mBase_29jun_abstract.csv\n"
     ]
    }
   ],
   "source": [
    "str_database_name = 'mBase_29jun.xlsx'\n",
    "str_version_db_name = 'mBase_29jun'\n",
    "csv_output_file_name = str_version_db_name + \"_abstract.csv\"\n",
    "print(csv_output_file_name)\n",
    "\n",
    "milk_df = pd.read_excel(str_database_name,sheet_name='Source')\n",
    "milk_df = milk_df.dropna(subset=['Source_DOI'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Source</th>\n",
       "      <th>Title</th>\n",
       "      <th>Source_DOI</th>\n",
       "      <th>PublDate</th>\n",
       "      <th>InPutDate</th>\n",
       "      <th>InputBy</th>\n",
       "      <th>Finished</th>\n",
       "      <th>is_useful</th>\n",
       "      <th>is_supplementary</th>\n",
       "      <th>Comment / Summary</th>\n",
       "      <th>Milk</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Andres_12</td>\n",
       "      <td>Developmental Status of 1-year-old Infants Fed...</td>\n",
       "      <td>https://doi.org/10.1542/peds.2011-3121</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>Chyanne</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>No Human Milk Composition - Only states that i...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Calvo-Castro_19</td>\n",
       "      <td>Pharmacokinetics of Vitamin E, γ-Oryzanol, and...</td>\n",
       "      <td>https://doi.org/10.1007/s00394-018-1770-6</td>\n",
       "      <td>2019.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>Chyanne</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>No Human Milk Composition - Only compositions ...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Dandrifosse_00</td>\n",
       "      <td>Are milk polyamines preventive agents against ...</td>\n",
       "      <td>https://doi.org/10.1017/s0029665100000100</td>\n",
       "      <td>2000.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>Chyanne</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Review (descriptive)  about spermine and sperm...</td>\n",
       "      <td>HumanMilk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Deep_12</td>\n",
       "      <td>Angiopreventive Efficacy of Pure Flavonolignan...</td>\n",
       "      <td>https://doi.org/10.1371/journal.pone.0034630</td>\n",
       "      <td>2012.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>Chyanne</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>No Human Milk Composition - Only composition o...</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Dunstan_04</td>\n",
       "      <td>The effect of supplementation with fish oil du...</td>\n",
       "      <td>https://doi.org/10.1111/j.1365-2222.2004.02028.x</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>2020.0</td>\n",
       "      <td>Chyanne</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>cytokines, Iga, and sCD14</td>\n",
       "      <td>HumanMilk</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>361</th>\n",
       "      <td>Gibbs_77</td>\n",
       "      <td>Drip Breast Milk: It's Composition, Collection...</td>\n",
       "      <td>https://doi.org/10.1016/0378-3782(77)90037-8</td>\n",
       "      <td>1977.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tünde</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>362</th>\n",
       "      <td>Goehring_16</td>\n",
       "      <td>Similar to Those Who Are Breastfed, Infants Fe...</td>\n",
       "      <td>https://doi.org/10.3945/jn.116.236919</td>\n",
       "      <td>2016.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tünde</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>363</th>\n",
       "      <td>Goelz_09</td>\n",
       "      <td>Effects of Different CMV-heat-inactivation-met...</td>\n",
       "      <td>https://doi.org/10.1203/PDR.0b013e3181991f18</td>\n",
       "      <td>2009.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tünde</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>364</th>\n",
       "      <td>Goes_02</td>\n",
       "      <td>Nutrient Composition of Banked Human Milk in B...</td>\n",
       "      <td>https://doi.org/10.1016/s0899-9007(02)00813-4</td>\n",
       "      <td>2002.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Tünde</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>366</th>\n",
       "      <td>Ahmed_04</td>\n",
       "      <td>Antioxidant Micronutrient Profile (Vitamin E, ...</td>\n",
       "      <td>https://doi.org/10.1093/tropej/50.6.357</td>\n",
       "      <td>2004.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>363 rows × 11 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              Source                                              Title  \\\n",
       "0          Andres_12  Developmental Status of 1-year-old Infants Fed...   \n",
       "1    Calvo-Castro_19  Pharmacokinetics of Vitamin E, γ-Oryzanol, and...   \n",
       "2     Dandrifosse_00  Are milk polyamines preventive agents against ...   \n",
       "3            Deep_12  Angiopreventive Efficacy of Pure Flavonolignan...   \n",
       "4         Dunstan_04  The effect of supplementation with fish oil du...   \n",
       "..               ...                                                ...   \n",
       "361         Gibbs_77  Drip Breast Milk: It's Composition, Collection...   \n",
       "362      Goehring_16  Similar to Those Who Are Breastfed, Infants Fe...   \n",
       "363         Goelz_09  Effects of Different CMV-heat-inactivation-met...   \n",
       "364          Goes_02  Nutrient Composition of Banked Human Milk in B...   \n",
       "366         Ahmed_04  Antioxidant Micronutrient Profile (Vitamin E, ...   \n",
       "\n",
       "                                           Source_DOI  PublDate  InPutDate  \\\n",
       "0              https://doi.org/10.1542/peds.2011-3121    2012.0     2020.0   \n",
       "1           https://doi.org/10.1007/s00394-018-1770-6    2019.0     2020.0   \n",
       "2           https://doi.org/10.1017/s0029665100000100    2000.0     2020.0   \n",
       "3        https://doi.org/10.1371/journal.pone.0034630    2012.0     2020.0   \n",
       "4    https://doi.org/10.1111/j.1365-2222.2004.02028.x    2004.0     2020.0   \n",
       "..                                                ...       ...        ...   \n",
       "361      https://doi.org/10.1016/0378-3782(77)90037-8    1977.0        NaN   \n",
       "362             https://doi.org/10.3945/jn.116.236919    2016.0        NaN   \n",
       "363      https://doi.org/10.1203/PDR.0b013e3181991f18    2009.0        NaN   \n",
       "364     https://doi.org/10.1016/s0899-9007(02)00813-4    2002.0        NaN   \n",
       "366           https://doi.org/10.1093/tropej/50.6.357    2004.0        NaN   \n",
       "\n",
       "     InputBy  Finished  is_useful  is_supplementary  \\\n",
       "0    Chyanne       1.0        0.0               0.0   \n",
       "1    Chyanne       1.0        0.0               0.0   \n",
       "2    Chyanne       1.0        0.0               0.0   \n",
       "3    Chyanne       1.0        0.0               0.0   \n",
       "4    Chyanne       1.0        1.0               0.0   \n",
       "..       ...       ...        ...               ...   \n",
       "361    Tünde       NaN        NaN               NaN   \n",
       "362    Tünde       NaN        NaN               NaN   \n",
       "363    Tünde       NaN        NaN               NaN   \n",
       "364    Tünde       NaN        NaN               NaN   \n",
       "366      NaN       NaN        NaN               NaN   \n",
       "\n",
       "                                     Comment / Summary       Milk  \n",
       "0    No Human Milk Composition - Only states that i...        NaN  \n",
       "1    No Human Milk Composition - Only compositions ...        NaN  \n",
       "2    Review (descriptive)  about spermine and sperm...  HumanMilk  \n",
       "3    No Human Milk Composition - Only composition o...        NaN  \n",
       "4                            cytokines, Iga, and sCD14  HumanMilk  \n",
       "..                                                 ...        ...  \n",
       "361                                                NaN        NaN  \n",
       "362                                                NaN        NaN  \n",
       "363                                                NaN        NaN  \n",
       "364                                                NaN        NaN  \n",
       "366                                                NaN        NaN  \n",
       "\n",
       "[363 rows x 11 columns]"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "milk_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_dois = milk_df['Source_DOI'].tolist()\n",
    "dois = set()\n",
    "pmids = set()\n",
    "urls_not_in_pubmed = set()\n",
    "for raw_doi in raw_dois:\n",
    "    if 'doi.' in raw_doi:\n",
    "        doi = raw_doi.split('org/')[1].strip()\n",
    "        dois.add(doi)\n",
    "    elif 'PMID:' in raw_doi:\n",
    "        pmid = raw_doi.split(':')[1].strip()\n",
    "        pmids.add(pmid)\n",
    "    else:\n",
    "        urls_not_in_pubmed.add(raw_doi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "358"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dois) + len(pmids) + len(urls_not_in_pubmed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'13099905', '2260791', '3372922', '3808849', '8920139', '9557492'}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pmids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'-',\n",
       " 'http://www.mcdb.ca/metabolites',\n",
       " 'https://assets.publishing.service.gov.uk/government/uploads/system/uploads/attachment_data/file/743819/The_Composition_of_Mature_Human_Milk__1977_.pdf',\n",
       " 'https://hmdb.ca/metabolites',\n",
       " 'https://pediatrics.aappublications.org/content/91/3/637',\n",
       " 'https://www.ars.usda.gov/northeast-area/beltsville-md-bhnrc/beltsville-human-nutrition-research-center/food-surveys-research-group/docs/fndds-download-databases/',\n",
       " 'https://www.fasebj.org/doi/abs/10.1096/fasebj.23.1_supplement.lb448'}"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "urls_not_in_pubmed"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "import urllib.request as request\n",
    "from lxml import etree\n",
    "import math\n",
    "\n",
    "# Constructs appropriate url for pubmed api from search terms\n",
    "def construct_url(url_input, query_type):\n",
    "    # Constructs url for search query\n",
    "    if query_type == 'search':\n",
    "        base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/esearch.fcgi?db=pubmed&term='\n",
    "        if isinstance(url_input, str):\n",
    "            term_url = url_input.replace(\" \", \"%20\")\n",
    "        else:\n",
    "            adjusted_terms = [s.replace(\" \", \"%20\") for s in url_input]\n",
    "            term_url = '%20AND%20'.join(adjusted_terms)\n",
    "        url = base_url + term_url\n",
    "        return url\n",
    "    elif query_type == 'document':\n",
    "        base_url = 'https://eutils.ncbi.nlm.nih.gov/entrez/eutils/efetch.fcgi?db=pubmed&id='\n",
    "        doc_urls = \"\"\n",
    "        for i in url_input:\n",
    "            if isinstance(i, str): \n",
    "                doc_urls = doc_urls + \",\" + i\n",
    "            else:\n",
    "                doc_urls = doc_urls + \",\" + str(i)\n",
    "        url = base_url + doc_urls.lstrip(\",\") + '&retmode=xml'\n",
    "        return url\n",
    "\n",
    "# Divides doc ids for larger paper queries in retrieve_doc_info()\n",
    "def divide_list(ids, num_divisions):\n",
    "    split_ids = np.array_split(list(ids), num_divisions)\n",
    "    split_ids = [np.ndarray.tolist(split_ids[i]) for i in range(len(split_ids))]\n",
    "\n",
    "    return split_ids\n",
    "\n",
    "# Enters search terms into pubmed database to return document ID's\n",
    "def search_pubmed(search_term):\n",
    "    url = construct_url(search_term, 'search')\n",
    "    with request.urlopen(url) as response:\n",
    "        xml = response.read()\n",
    "    root = etree.fromstring(xml)\n",
    "    # Recursively gets all objects where the tag is Id\n",
    "    if (root.findall('.//Count') is None) | (root.findall('.//Count')[0].text == '0'):\n",
    "        return np.nan, np.nan\n",
    "    elements = root.findall('.//Count')\n",
    "    ID = root.findall('.//Id')[0].text\n",
    "    # Converts all lxml objects to their text values\n",
    "    ids = [i.text for i in elements]\n",
    "    return ids[0], ID\n",
    "\n",
    "# Retrieves document (paper) info using pubmed paper ids\n",
    "def retrieve_doc_info(ids):\n",
    "    # Can't query too much in a single query, so divides larger id lists into seperate queries\n",
    "    num_loops = int(math.ceil(len(ids) / 100))\n",
    "    # Have to split requests larger than 100 documents to keep it within url size\n",
    "    ids = divide_list(ids, num_loops)\n",
    "    documents = []\n",
    "    # Retrieves xml data from pubmed\n",
    "    for i in ids:\n",
    "        url = construct_url(i, 'document')\n",
    "        with request.urlopen(url) as response:\n",
    "            xml = response.read()\n",
    "        root = etree.fromstring(xml)\n",
    "        documents = documents + root.findall('PubmedArticle')\n",
    "    info = pd.DataFrame()\n",
    "    for document in documents:\n",
    "        doc_id = int(document.find('.//PMID').text)\n",
    "        paper = document.find('.//ArticleTitle').text\n",
    "        journal = document.find('.//Title').text\n",
    "        year = document.find('.//Year').text\n",
    "\n",
    "        if document.find('.//AbstractText') is not None:\n",
    "            abstract = document.find('.//AbstractText').text\n",
    "        else:\n",
    "            abstract = None\n",
    "        new_row = {\n",
    "            'PMID' : doc_id,\n",
    "            'Paper' : paper,\n",
    "            'Journal' : journal,\n",
    "            'Year' : year,\n",
    "            'Abstract' : abstract,\n",
    "            'Webpage' : 'https://www.ncbi.nlm.nih.gov/pubmed/' + str(doc_id)\n",
    "        }\n",
    "        info = info.append(new_row, ignore_index = True)\n",
    "    info['PMID'] = info['PMID'].astype('int32')\n",
    "    return info.reset_index(drop = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "import time\n",
    "\n",
    "unknown_dois = set()\n",
    "# NCBI requirement: At most three queries per second if no API key is provided.\n",
    "# Equivalently, at least a third of second between queries\n",
    "delay = 0.33333333\n",
    "for doi in dois:\n",
    "    time.sleep(delay)\n",
    "    pmid = search_pubmed(doi)[1]\n",
    "    if str(pmid) != 'nan':\n",
    "        pmids.add(pmid)\n",
    "    else:\n",
    "        unknown_dois.add(doi)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'10200140',\n",
       " '10353922',\n",
       " '10400108',\n",
       " '10489525',\n",
       " '10569230',\n",
       " '10652985',\n",
       " '10673912',\n",
       " '10828177',\n",
       " '10837303',\n",
       " '10940350',\n",
       " '11234918',\n",
       " '11242449',\n",
       " '1130835',\n",
       " '11425797',\n",
       " '11787695',\n",
       " '11787710',\n",
       " '11787720',\n",
       " '11927700',\n",
       " '11994712',\n",
       " '12225305',\n",
       " '12496220',\n",
       " '12499346',\n",
       " '12548290',\n",
       " '12603709',\n",
       " '12812151',\n",
       " '12911694',\n",
       " '13099905',\n",
       " '1331178',\n",
       " '1442650',\n",
       " '14576748',\n",
       " '14599042',\n",
       " '14657823',\n",
       " '14973953',\n",
       " '14991268',\n",
       " '15097435',\n",
       " '15295084',\n",
       " '15298564',\n",
       " '15343178',\n",
       " '15384616',\n",
       " '15537722',\n",
       " '15585800',\n",
       " '15623823',\n",
       " '15681631',\n",
       " '15693764',\n",
       " '15693858',\n",
       " '15736916',\n",
       " '15867329',\n",
       " '16011458',\n",
       " '16061595',\n",
       " '16203669',\n",
       " '16261995',\n",
       " '16280560',\n",
       " '16306212',\n",
       " '16325533',\n",
       " '16390829',\n",
       " '16510619',\n",
       " '16512938',\n",
       " '16614484',\n",
       " '17002410',\n",
       " '17073150',\n",
       " '17177685',\n",
       " '17211132',\n",
       " '17295795',\n",
       " '17311985',\n",
       " '17443195',\n",
       " '17556680',\n",
       " '17596738',\n",
       " '17661565',\n",
       " '17761135',\n",
       " '17852471',\n",
       " '17915960',\n",
       " '17941914',\n",
       " '17942378',\n",
       " '17957152',\n",
       " '18053685',\n",
       " '18088092',\n",
       " '1808991',\n",
       " '18192331',\n",
       " '18221472',\n",
       " '18436967',\n",
       " '18477059',\n",
       " '18519456',\n",
       " '18547467',\n",
       " '1867111',\n",
       " '18947973',\n",
       " '18952725',\n",
       " '19034508',\n",
       " '19109654',\n",
       " '19127217',\n",
       " '19366315',\n",
       " '19444237',\n",
       " '19581827',\n",
       " '19807706',\n",
       " '19841193',\n",
       " '1989413',\n",
       " '20008425',\n",
       " '20105254',\n",
       " '20105662',\n",
       " '20172207',\n",
       " '20372777',\n",
       " '20394371',\n",
       " '20409714',\n",
       " '20639776',\n",
       " '20668239',\n",
       " '20679197',\n",
       " '20721564',\n",
       " '20801561',\n",
       " '20831709',\n",
       " '20850762',\n",
       " '21126862',\n",
       " '21133381',\n",
       " '21330996',\n",
       " '21524629',\n",
       " '21587097',\n",
       " '21592688',\n",
       " '21622093',\n",
       " '21695057',\n",
       " '2181825',\n",
       " '21921811',\n",
       " '219681',\n",
       " '22085741',\n",
       " '22099335',\n",
       " '22123889',\n",
       " '22128211',\n",
       " '22131550',\n",
       " '2239784',\n",
       " '2246719',\n",
       " '22513036',\n",
       " '22513202',\n",
       " '22514647',\n",
       " '22563219',\n",
       " '2260791',\n",
       " '22641754',\n",
       " '22652586',\n",
       " '22676802',\n",
       " '22805009',\n",
       " '22836031',\n",
       " '2284162',\n",
       " '22973420',\n",
       " '22974824',\n",
       " '22990856',\n",
       " '23178060',\n",
       " '23237648',\n",
       " '23265309',\n",
       " '23271066',\n",
       " '23274339',\n",
       " '23577672',\n",
       " '23752081',\n",
       " '23861770',\n",
       " '23892749',\n",
       " '2395063',\n",
       " '23968744',\n",
       " '24102779',\n",
       " '24376898',\n",
       " '24411973',\n",
       " '24423505',\n",
       " '24473952',\n",
       " '25174435',\n",
       " '25208800',\n",
       " '25221936',\n",
       " '25432966',\n",
       " '25527660',\n",
       " '25529691',\n",
       " '25651488',\n",
       " '25726238',\n",
       " '25766796',\n",
       " '25855820',\n",
       " '26061885',\n",
       " '26886782',\n",
       " '26907090',\n",
       " '27075905',\n",
       " '27136582',\n",
       " '27155888',\n",
       " '27164830',\n",
       " '27217155',\n",
       " '27251401',\n",
       " '27298082',\n",
       " '27490567',\n",
       " '27690094',\n",
       " '27702672',\n",
       " '27798337',\n",
       " '2782063',\n",
       " '27827913',\n",
       " '27830423',\n",
       " '27903517',\n",
       " '28194267',\n",
       " '28202638',\n",
       " '28244773',\n",
       " '28251339',\n",
       " '28301837',\n",
       " '28347466',\n",
       " '28538696',\n",
       " '28777356',\n",
       " '28880568',\n",
       " '2918394',\n",
       " '29344587',\n",
       " '29353262',\n",
       " '29470421',\n",
       " '29549961',\n",
       " '29686862',\n",
       " '29931273',\n",
       " '29958393',\n",
       " '29978378',\n",
       " '2998180',\n",
       " '2999182',\n",
       " '30051170',\n",
       " '30247884',\n",
       " '30311153',\n",
       " '30513944',\n",
       " '306224',\n",
       " '30643289',\n",
       " '30682200',\n",
       " '30706708',\n",
       " '30717735',\n",
       " '30785913',\n",
       " '30818777',\n",
       " '30886898',\n",
       " '30913698',\n",
       " '30984547',\n",
       " '30988385',\n",
       " '30994344',\n",
       " '3103547',\n",
       " '31109612',\n",
       " '31137596',\n",
       " '31152543',\n",
       " '31192916',\n",
       " '31336991',\n",
       " '31438647',\n",
       " '31581741',\n",
       " '31589250',\n",
       " '31709968',\n",
       " '31814423',\n",
       " '31964441',\n",
       " '32074402',\n",
       " '32167634',\n",
       " '32192176',\n",
       " '32208690',\n",
       " '32267727',\n",
       " '32283198',\n",
       " '32293945',\n",
       " '32329067',\n",
       " '32401302',\n",
       " '32401307',\n",
       " '32422857',\n",
       " '32437424',\n",
       " '32438370',\n",
       " '32443751',\n",
       " '32450886',\n",
       " '32466906',\n",
       " '32473807',\n",
       " '32595044',\n",
       " '3372922',\n",
       " '3421201',\n",
       " '3439943',\n",
       " '3735009',\n",
       " '3751158',\n",
       " '3808849',\n",
       " '3944733',\n",
       " '3954320',\n",
       " '3966414',\n",
       " '3974167',\n",
       " '3984932',\n",
       " '3998870',\n",
       " '4054998',\n",
       " '645628',\n",
       " '6496384',\n",
       " '6502376',\n",
       " '6695830',\n",
       " '6788913',\n",
       " '6846220',\n",
       " '7058078',\n",
       " '7101285',\n",
       " '7125685',\n",
       " '7211370',\n",
       " '7234741',\n",
       " '7236122',\n",
       " '7315290',\n",
       " '7827005',\n",
       " '7913291',\n",
       " '7919750',\n",
       " '8111168',\n",
       " '8154907',\n",
       " '8827106',\n",
       " '8920139',\n",
       " '8979282',\n",
       " '9093980',\n",
       " '9164160',\n",
       " '9445498',\n",
       " '9522962',\n",
       " '9557492',\n",
       " '9691151',\n",
       " '9764891',\n",
       " '9781307',\n",
       " '9797640',\n",
       " '9925131'}"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pmids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'10.1002/(SICI)1098-2345(1997)41:3<195::AID-AJP3>3.0.CO;2-S',\n",
       " '10.1002/(SICI)1521-3803(19990801)43:4<233::AID-FOOD233>3.0.CO;2-B',\n",
       " '10.1016/0140-6736(92)92684-8',\n",
       " '10.1016/0163-7827(95)00010-0',\n",
       " '10.1016/0377-8401(96)00973-X',\n",
       " '10.1016/0378-3782(77)90037-8',\n",
       " '10.1016/0378-3782(80)90003-1',\n",
       " '10.1016/0378-3782(81)90051-7',\n",
       " '10.1016/0378-3782(84)90096-3',\n",
       " '10.1016/0378-3782(92)90191-i',\n",
       " '10.1016/S0022-3476(05)81238-7',\n",
       " '10.1016/S0031-3955(05)70285-6',\n",
       " '10.1016/S0031-3955(05)70286-8',\n",
       " '10.1016/S0031-3955(05)70295-9',\n",
       " '10.1016/S0045-6535(99)00360-4',\n",
       " '10.1016/S0091-6749(03)01941-9',\n",
       " '10.1016/S0091-6749(99)70021-7',\n",
       " '10.1016/S0169-409X(03)00033-4',\n",
       " '10.1016/S0271-5317(87)80171-9',\n",
       " '10.1016/S0271-5317(98)00153-5',\n",
       " '10.1016/S0301-6226(01)00196-8',\n",
       " '10.1016/S0378-3782(97)00051-0',\n",
       " '10.1016/S0946-672X(98)80005-9',\n",
       " '10.1016/S0958-6946(02)00120-6',\n",
       " '10.1016/s0022-3476(05)81239-9',\n",
       " '10.1016/s0022-3476(77)80759-2',\n",
       " '10.1016/s0022-3476(77)80885-8',\n",
       " '10.1016/s0022-3476(80)80729-3',\n",
       " '10.1016/s0022-3476(80)80738-4',\n",
       " '10.1016/s0022-3476(81)80275-2',\n",
       " '10.1016/s0022-3476(86)80697-7',\n",
       " '10.1016/s0022-3476(87)80394-3',\n",
       " '10.1016/s0022-3476(98)70323-3',\n",
       " '10.1016/s0305-0491(99)00112-1',\n",
       " '10.1016/s0378-4347(97)00439-8',\n",
       " '10.1016/s0899-9007(02)00813-4',\n",
       " '10.1051/rnd:2004052',\n",
       " '10.1097/00005176-200000002-00003',\n",
       " '10.1289/ehp.116-a426',\n",
       " '10.1590/S0044-59672010000100008',\n",
       " '10.2223/JPED.1284',\n",
       " '10.3168/jds.S0022-0302(02)74079-4',\n",
       " '10.3168/jds.S0022-0302(03)73715-1',\n",
       " '10.3168/jds.S0022-0302(06)72198-1',\n",
       " '10.3168/jds.S0022-0302(91)78509-3',\n",
       " 'https://doi.'}"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "unknown_dois"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "manual_pmids = set([9057965, 10481820, 1357244, 9039426, 363402, 7389626, 7249996, 6714135, 1396267, 1560328, 11236733, 11236734, 11236727, 10739053, 14564350, 10589009, 12706546, 9363415, 9857329, 1560329, 318682, 336859, 7359266, 7359271, 7277107, 3772659, 3495653, 9602205, 10631803, 12093436, 15636165, 11913692, 12741556, 16537962, 1779072])\n",
    "dois_not_in_pubmed = ['10.1016/0377-8401(96)00973-X',\n",
    "    '10.1016/S0271-5317(87)80171-9',\n",
    "    '10.1016/S0271-5317(98)00153-5',\n",
    "    '10.1016/S0301-6226(01)00196-8',\n",
    "    '10.1016/S0958-6946(02)00120-6',\n",
    "    '10.1016/s0378-4347(97)00439-8',\n",
    "    '10.1097/00005176-200000002-00003',\n",
    "    '10.1289/ehp.116-a426',\n",
    "    '10.1590/S0044-59672010000100008',\n",
    "    '10.2223/JPED.1284',\n",
    "    'https://doi.'\n",
    "]\n",
    "manual_abstracts = [\n",
    "    \"This study aimed to examine the carbohydrate content (monosaccharides, lactose, and oligosaccharides) of human milk over 4 months of lactation to determine whether any changes occurred over time. Milk samples from 46 mothers, who delivered at term, were collected at 4th, 10th, 30th, 60th, 90th, and 120th days after delivery. Carbohydrates were measured by high-pressure liquid chromatography. Mean lactose concentration (±SD) increased from 56 ± 6.06 g/L on day 4 to 68.9 ± 8.16 g/L on day 120. Oligosaccharide level decreased from 20.9 ± 4.81 g/L to 12.9 ± 3.30 g/L, respectively. Monosaccharides represented only 1.2% of total carbohydrates. The changes in carbohydrate composition found indicate that carbohydrate synthesis by the mammary gland is a dynamic process. The physiological and biological relevance of human milk oligosaccharides is also discussed.\",\n",
    "    \"This study represents analysis of 220 human milk samples from nine cities in nine countries crossing five continents. Samples were collected from mothers from 18 to 40 years of age with a lactation range from 30 days to 188 days. Amino acids were analyzed using Waters Accq_Tag HPLC method. Total protein was calculated from the nitrogen content multiplied by a protein factor of 6.25. The nitrogen content was analyzed using FP‐528 nitrogen analyzer made by LECO Corporation. True protein was calculated from amino acid analysis and includes protein, free amino acids and peptides. Total protein, true protein, and the amino acid composition are not influenced by mother's age, and are statistically similar among locations, with the exception of samples from Chile, which had higher total protein and true protein. True protein is approximately 76% of total protein, is statistically similar among locations, and is not influenced by mother's age or days of lactation. Total protein, true protein, and individual amino acid concentrations in human milk steadily decline from 30 to 151 days of lactation, and are significantly higher in the second month than in the following four months. The amino acid profiles are statistically similar among locations. These results confirm higher protein and amino acid levels in early lactation and demonstrate consistency in the protein and amino acid composition of human milk across all locations.\",\n",
    "    \"The fatty acid composition of bovine milk fat can be substantially altered by feeding lipid sources which alter the fatty acid profile of lipid entering the intestine from the rumen. As long-chain fatty acids of dietary origin can be incorporated directly into milk fat the opportunity exists to alter the ratio of short and long-chain fatty acids as well as the degree of saturation of milk fat. In practice our ability to alter the fatty acid profile of milk fat is limited not by the synthetic capacity of the mammary gland, but rather by the challenge of achieving effective protection of unsaturated dietary fatty acids from biohydrogenation in the rumen, as well as keeping the level of polyunsaturated fatty acids within the range where the organoleptic quality and shelf-life of milk and dairy products are not compromised. The fatty acid composition of oilseeds such as canola are considered desirable from a human health perspective and thus their inclusion in the diet of dairy cattle as a means of achieving a more desirable fatty acid profile in milk fat may enhance the nutritive quality of milk.\",\n",
    "    \"Although the biochemistry of milk from mothers giving birth prematurely has been described in some detail, it was not known whether premature birth would affect vitamin D status of mothers, the vitamin D activity of milk or the concentration of vitamin D binding protein in milk. In the present study, plasma and milk samples were collected from eight mothers giving birth prematurely and compared to samples obtained from nine mothers giving birth at term. Vitamin D intake from milk and vitamin D supplements was at least 400 IU/day in all mothers. Complete 24-hour expressions of milk were collected between 14 and 21 days postpartum. No differences were observed in plasma D2 (5.1+0.7 and 3.7+1.0 ng/ml, mean±SEM) or milk D2 (0.19±0.06 and 0.12±0.02 ng/ml) in term versus preterm groups, respectively. In contrast, plasma vitamin D3 (0.7±0.1 and 2.7±0.5, ng/ml, p<0.01) and milk vitamin D3 (0.14±0.02 and 0.23±0.03, ng/ml, p<0.05) were significantly higher in preterm mothers. This was associated with higher plasma vitamin D binding protein (DBP) in preterm mothers (308±13 versus 463±30, ug/ml, p<0.01) but not preterm milk DBP. There were no differences in milk vitamin D sterols nor DBP content between the second and fourth week of lactation. In the case of the hydroxylated sterols, 25-hydroxyvitamin D2 and 25-hydroxyvitamin D3, no differences between mother groups could be discerned in plasma or milk. Differences in the synthesis and/or transport of vitamin D and DBP which are reflected in PT milk and plasma levels in the early postpartum period may relate to cutaneous synthesis or abbreviated gestational stage. Based on the vitamin D sterols measured in this study, the total vitamin D activity in human milk in the first month of lactation averaged 80±9 IU/L for preterm milk and 60±7 IU/L for term milk, both values which are in excess of the often quoted value for human milk vitamin D of 22 IU/L.\",\n",
    "    \"The purpose of this study was to determine the vitamin B12 status of vegetarian, lactating women who had immigrated to the US from India. Women exclusively breast-feeding their 4–20 week old infants were studied [8 lactovegetarians (VEG group), 11 omnivorous women (CONTROL group)]. Fasting maternal blood and breast milk samples were collected and analyzed for B12 concentrations; a random urine sample was collected from infants to determine methylmalonic acid (MMA) concentration. All subjects, except three VEG, reported taking vitamin supplements containing 2 to 25 μg of vitamin B12 regularly during pregnancy and lactation. Vitamin B12 intake from dietary sources and total intake (dietary plus supplement) was significantly lower in the VEG group compared to CONTROL subjects (1.35 ± 0.27 vs 2.61 ± 0.23, p<0.01 and 4.60 ± 1.43 vs 11.70 ± 2.47 μg/d, p = 0.04; respectively). Serum and milk vitamin B12 concentrations in VEG women were significantly lower than the CONTROL group (264 ± 52 vs 531 ± 78 pmol/L, p = 0.02 and 277 ± 39 vs 544 ± 104 pmol/L, p = 0.05; respectively). After controlling for diet, the mean milk vitamin B12 concentration of mothers at 16–20 wks postpartum was significantly lower than mothers 4–8 wks postpartum (p = 0.04). Assuming that infants consumed 750 ml of milk/d, 75% of infants of VEG mothers and 27% of infants of CONTROL mothers were not consuming the RDA for vitamin B12. All MMA concentrations were within normal limits except for one infant whose mother consumed a VEG diet. These results suggest the importance of supplementation with vitamin B12 during lactation among lactovegetarian women.\",\n",
    "    \"This review summarises the known effects of forages, animal fats or marine oils on bovine milk fat secretion and composition. Special attention is given to fatty acids that could play a positive role for human health, such as butyric acid, oleic acid, C18 to C22 polyunsaturated fatty acids and conjugated linoleic acid (CLA). The efficiency of the transfer of n-3 polyunsaturated fatty acids from diet to milk is reviewed. Milk fat from pasture fed cows seems to be higher in linolenic acid than milk fat from cows receiving preserved grass or maize, but the magnitude of this difference is limited. Indirect comparisons show that milk fat from maize silage diets is richer in short-chain FA and linoleic acid when compared to grass silage diets. Compared to fresh grass, grass silage favours myristic and palmitic acids at the expense of mono- and polyunsaturated FA, including CLA. Protected tallow allows for a large increase in milk fat yield, and in the percentage of milk stearic and oleic acids, at the expense of medium chain FA. Non-protected tallow has a similar effect on medium chain FA without increasing so much C18 FA yield, which explains that it does not increase milk fat yield. Dose–response curves of milk CLA are reviewed for marine oil supplements, as well as the relationship between milk CLA and trans-C18:1 contents. The potential of marine oil supplementation to increase the mean CLA content in cow milk fat is large (more than 300% above basal values). A specific role for dietary C20:5 n-3 in the sharp decrease in milk fat secretion after fish oil supplementation is suggested. However, there is a need to evaluate how the different feeding strategies could change the other aspects of milk fat quality, such as taste, oxidative stability or manufacturing value.\",\n",
    "    \"Milk composition of mammallian species varies widely with reference to genetic, physiological and nutritional factors and environmental conditions. In this survey, the composition of mare's milk is reviewed and compared to human and cow's milk, considering principal protein fractions and fatty acid content. Protein content in mare's milk is higher than in human milk and lower than in cow's milk; casein concentration in mare's milk is intermediate between the other two milks. Fat content is lower in mare's milk compared to human and cow's milk. Distribution of di- and tri-glycerides in mare's and women’ milk is similar. The proportion of polyunsaturated fatty acids in mare's and human milk is remarkably higher than in cow's milk. Mare's milk shows some structural and functional peculiarities that make it more suitable for human nourishment than cow's milk.\",\n",
    "    \"Pooled human milk was subjected to Holder pasteurization and storage at −20°C up to 90 days and examined for its content of fat and l-lactate and for lipid composition. This treatment reduced fats by 6% and l-lactate by at least 7%. In addition, pasteurization and storage induced triglyceride hydrolysis. The absolute amount of free fatty acids (FFAs) which was 0.5% after collection, doubled after pasteurization and rose even more after storage. Different FFA compositions were found by several authors using the same analytical method even for milk samples subjected to the same treatment. More detailed information on procedures must be given to explain the different results.\",\n",
    "    \"When it comes to feeding the newborn, human milk is, from an evolutionary perspective, the biological norm, the time-tested standard of care. The health benefits to the infant of breast-feeding have been amply documented; numerous studies strongly indicate significantly decreased risks of infection, allergy, asthma, arthritis, diabetes, obesity, cardiovascular disease, and various cancers in both childhood and adulthood. Among the more fundamental disadvantages of not being breastfed is a loss of immunologic protection afforded by maternal colostrum, a “pre-milk” fluid secreted only during the first days after delivery, as well as numerous other bioactive factors that help protect the infant through the first two years of life, when the immune and nervous systems are incompletely developed. Nevertheless, given the tendency for persistent organic pollutants (POPs), pesticides, heavy metals, and other contaminants to accumulate in human milk, researchers and parents alike are asking whether the nursling’s exposure to these pollutants might reduce or even override the health benefits.\",\n",
    "    \"This study treats a transversal, descriptive, observational and analytical study with the objective of determining the Vitamin A chemical content in Human Maternal Milk (HMM) of 50 donor mothers from the Ana Braga Maternity Human Milk Bank (HMB), in Manaus, Amazonas. Samples were collected weekly and maintained frozen until pasteurization. Analyses of retinol and β-carotene in human milk were undertaken before and after the treatment (pasteurization), chromatographic method. Aliquots of approximately 5 mL of milk were taken from each sample. The babies included in the study were weight > 2500g. The Vitamin A concentration in the milk, before and after the processing, was found to be insufficient for that required (46.57 ± 6.28 µg/dL and 31.55 ± 8.29 µg/dL respectively) for children who feed exclusively on HMB milk, notably premature babies, whose Vitamin A requirements are greater and whose capacity for ingesting milk is lower than that of fully born babies. The loss of Vitamin A through processing was, in this study, 32.51%. To guarantee hepatic reserves and protect against deficiencies, it would be necessary that the vitamin A concentration in the milk be higher than 80 µg/dL.\",\n",
    "    \"OBJECTIVES: To analyze the effect of processing on the Vitamin A levels of milk that is donated to the Human Milk Bank at the University Maternity Hospital, and to evaluate whether this milk supplies the infants' Vitamin A requirement. METHODS: Sixty milk samples from the Human Milk Bank were divided into two equal portions. One milk portion was extracted prior to processing, while another fraction was subjected to pasteurization and later extracted. The samples were analyzed using high performance liquid chromatography. Analysis with Student's t test indicated that the difference between mean retinol levels was statistically significant at the p < 0.05 level. RESULTS: The retinol content found in the milk prior to processing was 55.4±34.0 µg/100 ml, whereas for the processed milk this level was 36.6±26.1 µg/100 ml (p < 0.001). CONCLUSION: It was found that retinol loss occurs during milk processing and that the milk from the Human Milk Bank does not meet infants' Vitamin A requirement.\"\n",
    "]\n",
    "\n",
    "pmids |= manual_pmids"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Journal</th>\n",
       "      <th>PMID</th>\n",
       "      <th>Paper</th>\n",
       "      <th>Webpage</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There has been a striking generational increas...</td>\n",
       "      <td>Nature medicine</td>\n",
       "      <td>30643289</td>\n",
       "      <td>Healthy infants harbor intestinal bacteria tha...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/30643289</td>\n",
       "      <td>2019</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For term infants, human milk provides adequate...</td>\n",
       "      <td>The Cochrane database of systematic reviews</td>\n",
       "      <td>14973953</td>\n",
       "      <td>Multicomponent fortified human milk for promot...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/14973953</td>\n",
       "      <td>2004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The overall beneficial effects of breastfeedin...</td>\n",
       "      <td>Pediatric allergy and immunology : official pu...</td>\n",
       "      <td>23577672</td>\n",
       "      <td>Relationship between breast milk sCD14, TGF-β1...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/23577672</td>\n",
       "      <td>2014</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Milk samples from 33 women from a rural area o...</td>\n",
       "      <td>The American journal of clinical nutrition</td>\n",
       "      <td>219681</td>\n",
       "      <td>Studies on the quality of breast milk during 2...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/219681</td>\n",
       "      <td>1979</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The aim of this study was to compare the actua...</td>\n",
       "      <td>Journal of perinatology : official journal of ...</td>\n",
       "      <td>19444237</td>\n",
       "      <td>Preterm infants fed fortified human milk recei...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/19444237</td>\n",
       "      <td>2009</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>325</th>\n",
       "      <td>Insufficient polyamine intake could play a rol...</td>\n",
       "      <td>The Proceedings of the Nutrition Society</td>\n",
       "      <td>10828177</td>\n",
       "      <td>Are milk polyamines preventive agents against ...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/10828177</td>\n",
       "      <td>2000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>326</th>\n",
       "      <td>Bodyweight gain of formula-fed babies is usual...</td>\n",
       "      <td>Pediatrics international : official journal of...</td>\n",
       "      <td>15693858</td>\n",
       "      <td>Lower energy content of human milk than calcul...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/15693858</td>\n",
       "      <td>2005</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>327</th>\n",
       "      <td>The aim of the present study was to identify a...</td>\n",
       "      <td>Annals of nutrition &amp; metabolism</td>\n",
       "      <td>28301837</td>\n",
       "      <td>Breast Milk Polyamines and Microbiota Interact...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/28301837</td>\n",
       "      <td>2018</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>328</th>\n",
       "      <td>The authors discovered some errors regarding r...</td>\n",
       "      <td>Cancer</td>\n",
       "      <td>25855820</td>\n",
       "      <td>Erratum: Borderud SP, Li Y, Burkhalter JE, She...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/25855820</td>\n",
       "      <td>2015</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>329</th>\n",
       "      <td>Cytokines secreted in human milk might play im...</td>\n",
       "      <td>The Journal of allergy and clinical immunology</td>\n",
       "      <td>14564350</td>\n",
       "      <td>TGF-beta in human milk is associated with whee...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/14564350</td>\n",
       "      <td>2003</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>330 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Abstract  \\\n",
       "0    There has been a striking generational increas...   \n",
       "1    For term infants, human milk provides adequate...   \n",
       "2    The overall beneficial effects of breastfeedin...   \n",
       "3    Milk samples from 33 women from a rural area o...   \n",
       "4    The aim of this study was to compare the actua...   \n",
       "..                                                 ...   \n",
       "325  Insufficient polyamine intake could play a rol...   \n",
       "326  Bodyweight gain of formula-fed babies is usual...   \n",
       "327  The aim of the present study was to identify a...   \n",
       "328  The authors discovered some errors regarding r...   \n",
       "329  Cytokines secreted in human milk might play im...   \n",
       "\n",
       "                                               Journal      PMID  \\\n",
       "0                                      Nature medicine  30643289   \n",
       "1          The Cochrane database of systematic reviews  14973953   \n",
       "2    Pediatric allergy and immunology : official pu...  23577672   \n",
       "3           The American journal of clinical nutrition    219681   \n",
       "4    Journal of perinatology : official journal of ...  19444237   \n",
       "..                                                 ...       ...   \n",
       "325           The Proceedings of the Nutrition Society  10828177   \n",
       "326  Pediatrics international : official journal of...  15693858   \n",
       "327                   Annals of nutrition & metabolism  28301837   \n",
       "328                                             Cancer  25855820   \n",
       "329     The Journal of allergy and clinical immunology  14564350   \n",
       "\n",
       "                                                 Paper  \\\n",
       "0    Healthy infants harbor intestinal bacteria tha...   \n",
       "1    Multicomponent fortified human milk for promot...   \n",
       "2    Relationship between breast milk sCD14, TGF-β1...   \n",
       "3    Studies on the quality of breast milk during 2...   \n",
       "4    Preterm infants fed fortified human milk recei...   \n",
       "..                                                 ...   \n",
       "325  Are milk polyamines preventive agents against ...   \n",
       "326  Lower energy content of human milk than calcul...   \n",
       "327  Breast Milk Polyamines and Microbiota Interact...   \n",
       "328  Erratum: Borderud SP, Li Y, Burkhalter JE, She...   \n",
       "329  TGF-beta in human milk is associated with whee...   \n",
       "\n",
       "                                          Webpage  Year  \n",
       "0    https://www.ncbi.nlm.nih.gov/pubmed/30643289  2019  \n",
       "1    https://www.ncbi.nlm.nih.gov/pubmed/14973953  2004  \n",
       "2    https://www.ncbi.nlm.nih.gov/pubmed/23577672  2014  \n",
       "3      https://www.ncbi.nlm.nih.gov/pubmed/219681  1979  \n",
       "4    https://www.ncbi.nlm.nih.gov/pubmed/19444237  2009  \n",
       "..                                            ...   ...  \n",
       "325  https://www.ncbi.nlm.nih.gov/pubmed/10828177  2000  \n",
       "326  https://www.ncbi.nlm.nih.gov/pubmed/15693858  2005  \n",
       "327  https://www.ncbi.nlm.nih.gov/pubmed/28301837  2018  \n",
       "328  https://www.ncbi.nlm.nih.gov/pubmed/25855820  2015  \n",
       "329  https://www.ncbi.nlm.nih.gov/pubmed/14564350  2003  \n",
       "\n",
       "[330 rows x 6 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results = retrieve_doc_info(pmids)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Journal</th>\n",
       "      <th>PMID</th>\n",
       "      <th>Paper</th>\n",
       "      <th>Webpage</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There has been a striking generational increas...</td>\n",
       "      <td>Nature medicine</td>\n",
       "      <td>30643289.0</td>\n",
       "      <td>Healthy infants harbor intestinal bacteria tha...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/30643289</td>\n",
       "      <td>2019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For term infants, human milk provides adequate...</td>\n",
       "      <td>The Cochrane database of systematic reviews</td>\n",
       "      <td>14973953.0</td>\n",
       "      <td>Multicomponent fortified human milk for promot...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/14973953</td>\n",
       "      <td>2004.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The overall beneficial effects of breastfeedin...</td>\n",
       "      <td>Pediatric allergy and immunology : official pu...</td>\n",
       "      <td>23577672.0</td>\n",
       "      <td>Relationship between breast milk sCD14, TGF-β1...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/23577672</td>\n",
       "      <td>2014.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Milk samples from 33 women from a rural area o...</td>\n",
       "      <td>The American journal of clinical nutrition</td>\n",
       "      <td>219681.0</td>\n",
       "      <td>Studies on the quality of breast milk during 2...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/219681</td>\n",
       "      <td>1979.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The aim of this study was to compare the actua...</td>\n",
       "      <td>Journal of perinatology : official journal of ...</td>\n",
       "      <td>19444237.0</td>\n",
       "      <td>Preterm infants fed fortified human milk recei...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/19444237</td>\n",
       "      <td>2009.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>Milk composition of mammallian species varies ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>Pooled human milk was subjected to Holder past...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>When it comes to feeding the newborn, human mi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>This study treats a transversal, descriptive, ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340</th>\n",
       "      <td>OBJECTIVES: To analyze the effect of processin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>341 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Abstract  \\\n",
       "0    There has been a striking generational increas...   \n",
       "1    For term infants, human milk provides adequate...   \n",
       "2    The overall beneficial effects of breastfeedin...   \n",
       "3    Milk samples from 33 women from a rural area o...   \n",
       "4    The aim of this study was to compare the actua...   \n",
       "..                                                 ...   \n",
       "336  Milk composition of mammallian species varies ...   \n",
       "337  Pooled human milk was subjected to Holder past...   \n",
       "338  When it comes to feeding the newborn, human mi...   \n",
       "339  This study treats a transversal, descriptive, ...   \n",
       "340  OBJECTIVES: To analyze the effect of processin...   \n",
       "\n",
       "                                               Journal        PMID  \\\n",
       "0                                      Nature medicine  30643289.0   \n",
       "1          The Cochrane database of systematic reviews  14973953.0   \n",
       "2    Pediatric allergy and immunology : official pu...  23577672.0   \n",
       "3           The American journal of clinical nutrition    219681.0   \n",
       "4    Journal of perinatology : official journal of ...  19444237.0   \n",
       "..                                                 ...         ...   \n",
       "336                                                NaN         NaN   \n",
       "337                                                NaN         NaN   \n",
       "338                                                NaN         NaN   \n",
       "339                                                NaN         NaN   \n",
       "340                                                NaN         NaN   \n",
       "\n",
       "                                                 Paper  \\\n",
       "0    Healthy infants harbor intestinal bacteria tha...   \n",
       "1    Multicomponent fortified human milk for promot...   \n",
       "2    Relationship between breast milk sCD14, TGF-β1...   \n",
       "3    Studies on the quality of breast milk during 2...   \n",
       "4    Preterm infants fed fortified human milk recei...   \n",
       "..                                                 ...   \n",
       "336                                                NaN   \n",
       "337                                                NaN   \n",
       "338                                                NaN   \n",
       "339                                                NaN   \n",
       "340                                                NaN   \n",
       "\n",
       "                                          Webpage    Year  \n",
       "0    https://www.ncbi.nlm.nih.gov/pubmed/30643289  2019.0  \n",
       "1    https://www.ncbi.nlm.nih.gov/pubmed/14973953  2004.0  \n",
       "2    https://www.ncbi.nlm.nih.gov/pubmed/23577672  2014.0  \n",
       "3      https://www.ncbi.nlm.nih.gov/pubmed/219681  1979.0  \n",
       "4    https://www.ncbi.nlm.nih.gov/pubmed/19444237  2009.0  \n",
       "..                                            ...     ...  \n",
       "336                                           NaN     NaN  \n",
       "337                                           NaN     NaN  \n",
       "338                                           NaN     NaN  \n",
       "339                                           NaN     NaN  \n",
       "340                                           NaN     NaN  \n",
       "\n",
       "[341 rows x 6 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "for abstract in manual_abstracts:\n",
    "    new_row = {\n",
    "        'PMID' : 'nan',\n",
    "        'Paper' : 'nan',\n",
    "        'Journal' : 'nan',\n",
    "        'Year' : 'nan',\n",
    "        'Abstract' : abstract,\n",
    "        'Webpage' : 'nan'\n",
    "    }\n",
    "    results = results.append(new_row, ignore_index = True)\n",
    "results.to_csv(csv_output_file_name, index=False)\n",
    "results = pd.read_csv(csv_output_file_name)\n",
    "results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Abstract</th>\n",
       "      <th>Journal</th>\n",
       "      <th>PMID</th>\n",
       "      <th>Paper</th>\n",
       "      <th>Webpage</th>\n",
       "      <th>Year</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>There has been a striking generational increas...</td>\n",
       "      <td>Nature medicine</td>\n",
       "      <td>30643289.0</td>\n",
       "      <td>Healthy infants harbor intestinal bacteria tha...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/30643289</td>\n",
       "      <td>2019.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>For term infants, human milk provides adequate...</td>\n",
       "      <td>The Cochrane database of systematic reviews</td>\n",
       "      <td>14973953.0</td>\n",
       "      <td>Multicomponent fortified human milk for promot...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/14973953</td>\n",
       "      <td>2004.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The overall beneficial effects of breastfeedin...</td>\n",
       "      <td>Pediatric allergy and immunology : official pu...</td>\n",
       "      <td>23577672.0</td>\n",
       "      <td>Relationship between breast milk sCD14, TGF-β1...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/23577672</td>\n",
       "      <td>2014.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Milk samples from 33 women from a rural area o...</td>\n",
       "      <td>The American journal of clinical nutrition</td>\n",
       "      <td>219681.0</td>\n",
       "      <td>Studies on the quality of breast milk during 2...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/219681</td>\n",
       "      <td>1979.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The aim of this study was to compare the actua...</td>\n",
       "      <td>Journal of perinatology : official journal of ...</td>\n",
       "      <td>19444237.0</td>\n",
       "      <td>Preterm infants fed fortified human milk recei...</td>\n",
       "      <td>https://www.ncbi.nlm.nih.gov/pubmed/19444237</td>\n",
       "      <td>2009.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>336</th>\n",
       "      <td>Milk composition of mammallian species varies ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>337</th>\n",
       "      <td>Pooled human milk was subjected to Holder past...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>338</th>\n",
       "      <td>When it comes to feeding the newborn, human mi...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>This study treats a transversal, descriptive, ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>340</th>\n",
       "      <td>OBJECTIVES: To analyze the effect of processin...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>341 rows × 6 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              Abstract  \\\n",
       "0    There has been a striking generational increas...   \n",
       "1    For term infants, human milk provides adequate...   \n",
       "2    The overall beneficial effects of breastfeedin...   \n",
       "3    Milk samples from 33 women from a rural area o...   \n",
       "4    The aim of this study was to compare the actua...   \n",
       "..                                                 ...   \n",
       "336  Milk composition of mammallian species varies ...   \n",
       "337  Pooled human milk was subjected to Holder past...   \n",
       "338  When it comes to feeding the newborn, human mi...   \n",
       "339  This study treats a transversal, descriptive, ...   \n",
       "340  OBJECTIVES: To analyze the effect of processin...   \n",
       "\n",
       "                                               Journal        PMID  \\\n",
       "0                                      Nature medicine  30643289.0   \n",
       "1          The Cochrane database of systematic reviews  14973953.0   \n",
       "2    Pediatric allergy and immunology : official pu...  23577672.0   \n",
       "3           The American journal of clinical nutrition    219681.0   \n",
       "4    Journal of perinatology : official journal of ...  19444237.0   \n",
       "..                                                 ...         ...   \n",
       "336                                                NaN         NaN   \n",
       "337                                                NaN         NaN   \n",
       "338                                                NaN         NaN   \n",
       "339                                                NaN         NaN   \n",
       "340                                                NaN         NaN   \n",
       "\n",
       "                                                 Paper  \\\n",
       "0    Healthy infants harbor intestinal bacteria tha...   \n",
       "1    Multicomponent fortified human milk for promot...   \n",
       "2    Relationship between breast milk sCD14, TGF-β1...   \n",
       "3    Studies on the quality of breast milk during 2...   \n",
       "4    Preterm infants fed fortified human milk recei...   \n",
       "..                                                 ...   \n",
       "336                                                NaN   \n",
       "337                                                NaN   \n",
       "338                                                NaN   \n",
       "339                                                NaN   \n",
       "340                                                NaN   \n",
       "\n",
       "                                          Webpage    Year  \n",
       "0    https://www.ncbi.nlm.nih.gov/pubmed/30643289  2019.0  \n",
       "1    https://www.ncbi.nlm.nih.gov/pubmed/14973953  2004.0  \n",
       "2    https://www.ncbi.nlm.nih.gov/pubmed/23577672  2014.0  \n",
       "3      https://www.ncbi.nlm.nih.gov/pubmed/219681  1979.0  \n",
       "4    https://www.ncbi.nlm.nih.gov/pubmed/19444237  2009.0  \n",
       "..                                            ...     ...  \n",
       "336                                           NaN     NaN  \n",
       "337                                           NaN     NaN  \n",
       "338                                           NaN     NaN  \n",
       "339                                           NaN     NaN  \n",
       "340                                           NaN     NaN  \n",
       "\n",
       "[341 rows x 6 columns]"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}