SPOKEsig API example.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Before generating SPOKEsigs you will need EHR data..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This code used EHR data from OMOP (tables: condition_occurrence, drug_exposure, and measurement)<br/>\n",
    "Only the person_id and *concept_id columns were exported from the OMOP tables.<br/>\n",
    "Lab tests were filtered to only include abnormal results."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Import modules"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import requests\n",
    "from requests.auth import HTTPBasicAuth\n",
    "from retrying import retry\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "base_url = 'https://spokeapi.ucsf.edu'\n",
    "end_point = '/v1/spoke_sig'\n",
    "usr = 'user1'\n",
    "psw = 'Ou0At7jXZJmt9oGP9fbGZ2TwstNFFE2l7Tru69HxnCI'\n",
    "url = base_url + end_point\n",
    "\n",
    "@retry(stop_max_attempt_number=50, wait_fixed=10000)\n",
    "def get_api_resp(clinical_variables):\n",
    "    return requests.get(url, params={'src':clinical_variables}, auth = HTTPBasicAuth(usr, psw), verify=False)\n",
    "\n",
    "def make_spoke_sigs(all_concept_df):\n",
    "    patients_seen, spoke_sigs = [], []\n",
    "    for patient, clinical_variables in all_concept_df[['Patient_Index', 'concept_id']].values:\n",
    "        result = get_api_resp(clinical_variables)\n",
    "        if result.status_code == 200:\n",
    "            if len(result.json()[0]['spoke_sig']) >0:\n",
    "                print(\"API call was successfull, spoke signature for patient %s is available in 'spoke_sig' variable\" % patient)\n",
    "                patients_seen.append(patient)\n",
    "                spoke_sigs.append(result.json()[0]['spoke_sig'])\n",
    "            else:\n",
    "                print('API call could not create a spoke signature for patient %s' % patient)\n",
    "        else:\n",
    "            print('API call was not successfull for patient %s, returned %s HTTP status code' % (patient, str(result.status_code)))    \n",
    "    return np.array(patients_seen), np.array(spoke_sigs)\n",
    "\n",
    "def load_patient_information(diag_filename, med_filename, lab_filename):\n",
    "    print('Reading diagnosis file ...')\n",
    "    diag_df = pd.read_csv(diag_filename, sep='\\t', header=0, index_col=False)\n",
    "    print('Reading medication file ...')\n",
    "    med_df = pd.read_csv(med_filename, sep='\\t', header=0, index_col=False)\n",
    "    print('Reading lab file ...')\n",
    "    lab_df = pd.read_csv(lab_filename, sep='\\t', header=0, index_col=False)\n",
    "    print('Merging patient tables')\n",
    "    all_concept_df = pd.concat((diag_df, med_df, lab_df), axis=0).drop_duplicates()\n",
    "    all_concept_df.loc[:,'concept_id'] = ['OMOP:%s' % concept_id for concept_id in all_concept_df.concept_id.values]\n",
    "    all_concept_df = all_concept_df.groupby('person_id')['concept_id'].apply(list).reset_index()\n",
    "    all_concept_df.loc[:,'Patient_Index'] = np.arange(len(all_concept_df))\n",
    "    return all_concept_df"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Load and filter patient data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Reading diagnosis file ...\n",
      "Reading medication file ...\n",
      "Reading lab file ...\n",
      "Merging patient tables\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>person_id</th>\n",
       "      <th>concept_id</th>\n",
       "      <th>Patient_Index</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>230130</td>\n",
       "      <td>[OMOP:35225058, OMOP:45556996, OMOP:44824029, ...</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>411126</td>\n",
       "      <td>[OMOP:44833044, OMOP:37202304, OMOP:44825717, ...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>547231</td>\n",
       "      <td>[OMOP:45600511, OMOP:35225339, OMOP:35209124, ...</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>742728</td>\n",
       "      <td>[OMOP:45587523, OMOP:44826854, OMOP:44832716, ...</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1291295</td>\n",
       "      <td>[OMOP:45601745, OMOP:45551542, OMOP:44829584, ...</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   person_id                                         concept_id  Patient_Index\n",
       "0     230130  [OMOP:35225058, OMOP:45556996, OMOP:44824029, ...              0\n",
       "1     411126  [OMOP:44833044, OMOP:37202304, OMOP:44825717, ...              1\n",
       "2     547231  [OMOP:45600511, OMOP:35225339, OMOP:35209124, ...              2\n",
       "3     742728  [OMOP:45587523, OMOP:44826854, OMOP:44832716, ...              3\n",
       "4    1291295  [OMOP:45601745, OMOP:45551542, OMOP:44829584, ...              4"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# filenames from SQL queries\n",
    "diag_filename = 'condition_occurrence.tsv'\n",
    "med_filename = 'drug_exposure.tsv'\n",
    "lab_filename = 'measurement.tsv'\n",
    "\n",
    "all_concept_df = load_patient_information(diag_filename, med_filename, lab_filename)\n",
    "all_concept_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Make SPOKEsigs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "API call was successfull, spoke signature for patient 0 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 1 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 2 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 3 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 4 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 5 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 6 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 7 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 8 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 9 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 10 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 11 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 12 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 13 is available in 'spoke_sig' variable\n",
      "API call was successfull, spoke signature for patient 14 is available in 'spoke_sig' variable\n"
     ]
    }
   ],
   "source": [
    "patients_seen, spoke_sigs = make_spoke_sigs(all_concept_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Save new patient information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# filter patients w/o SPOKEsigs\n",
    "if len(all_concept_df) > len(patients_seen):\n",
    "    all_concept_df = all_concept_df[all_concept_df.Patient_Index.isin(patients_seen)]\n",
    "    all_concept_df.loc[:,'Patient_Index'] = np.arange(len(all_concept_df))\n",
    "\n",
    "# save patient info and SPOKEsigs\n",
    "np.save('new_patient_spokesigs.npy', spoke_sigs, allow_pickle=False)\n",
    "all_concept_df.to_csv('new_patient_info.tsv', sep='\\t', header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}