serializer.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.0.3'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "from IPython.display import display, Markdown\n",
    "from tqdm import tqdm\n",
    "import csv\n",
    "pd.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_input = \"dd_data/20230802/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "hgnc_info = pd.read_csv('dd_data/HGNC_genes.txt', sep=\"\\t\")\n",
    "hgnc_mapper = {}\n",
    "for i, row in hgnc_info.iterrows():\n",
    "    uid = row[\"HGNC ID\"]\n",
    "    enz = row[\"Enzyme (EC) ID\"]\n",
    "    unip = row[\"UniProt accession\"]\n",
    "    if uid not in hgnc_mapper:\n",
    "        hgnc_mapper[uid] = {}\n",
    "        if type(unip) == str:\n",
    "            hgnc_mapper[uid][\"UNIPROTKB\"] = unip\n",
    "        if type(enz) == str:\n",
    "            hgnc_mapper[uid][\"ec_id\"] = enz\n",
    "                \n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Empty DataFrame\n",
       "Columns: []\n",
       "Index: [C0000097, C0000359, C0000610, C0000739, C0000873]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts = pd.read_csv(base_input + \"neo4j/import/CUIs.csv\")\n",
    "concepts = pd.DataFrame(index=concepts[\"CUI:ID\"].unique())\n",
    "concepts.index.name = \"id\"\n",
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>STN</th>\n",
       "      <th>DEF</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TUI:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>T012</th>\n",
       "      <td>Bird</td>\n",
       "      <td>A1.1.3.1.1.2</td>\n",
       "      <td>A vertebrate having a constant body temperatur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T014</th>\n",
       "      <td>Reptile</td>\n",
       "      <td>A1.1.3.1.1.5</td>\n",
       "      <td>A cold-blooded vertebrate having an external c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T015</th>\n",
       "      <td>Mammal</td>\n",
       "      <td>A1.1.3.1.1.4</td>\n",
       "      <td>A vertebrate having a constant body temperatur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T021</th>\n",
       "      <td>Fully Formed Anatomical Structure</td>\n",
       "      <td>A1.2.3</td>\n",
       "      <td>An anatomical structure in a fully formed orga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T022</th>\n",
       "      <td>Body System</td>\n",
       "      <td>A2.1.4.1</td>\n",
       "      <td>A complex of anatomical structures that perfor...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     name           STN  \\\n",
       "TUI:ID                                                    \n",
       "T012                                 Bird  A1.1.3.1.1.2   \n",
       "T014                              Reptile  A1.1.3.1.1.5   \n",
       "T015                               Mammal  A1.1.3.1.1.4   \n",
       "T021    Fully Formed Anatomical Structure        A1.2.3   \n",
       "T022                          Body System      A2.1.4.1   \n",
       "\n",
       "                                                      DEF  \n",
       "TUI:ID                                                     \n",
       "T012    A vertebrate having a constant body temperatur...  \n",
       "T014    A cold-blooded vertebrate having an external c...  \n",
       "T015    A vertebrate having a constant body temperatur...  \n",
       "T021    An anatomical structure in a fully formed orga...  \n",
       "T022    A complex of anatomical structures that perfor...  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "semantics = pd.read_csv(base_input + \"neo4j/import/TUIs.csv\", index_col=0)\n",
    "semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SUI:ID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>S0009776</th>\n",
       "      <td>Acid, 2-Aminohexanedioic</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S7249234</th>\n",
       "      <td>BR CAMP</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S11872577</th>\n",
       "      <td>cramps abdominal</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S14680596</th>\n",
       "      <td>Retained tissue after pregnancy loss</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>S3417882</th>\n",
       "      <td>Missed miscarriage</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                           name\n",
       "SUI:ID                                         \n",
       "S0009776               Acid, 2-Aminohexanedioic\n",
       "S7249234                                BR CAMP\n",
       "S11872577                      cramps abdominal\n",
       "S14680596  Retained tissue after pregnancy loss\n",
       "S3417882                     Missed miscarriage"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "terms = pd.read_csv(base_input + \"neo4j/import/SUIs.csv\", index_col=0)\n",
    "terms.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_11529/2703827308.py:1: DtypeWarning: Columns (2) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  codes = pd.read_csv(base_input + \"neo4j/import/CODEs.csv\", index_col=0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SAB</th>\n",
       "      <th>CODE</th>\n",
       "      <th>value:float</th>\n",
       "      <th>lowerbound:float</th>\n",
       "      <th>upperbound:float</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CodeID:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MTHSPL:J7A92W69L7</th>\n",
       "      <td>MTHSPL</td>\n",
       "      <td>J7A92W69L7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCI:C76777</th>\n",
       "      <td>NCI</td>\n",
       "      <td>C76777</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC:N07XX07</th>\n",
       "      <td>ATC</td>\n",
       "      <td>N07XX07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GS:1946</th>\n",
       "      <td>GS</td>\n",
       "      <td>1946</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NOC:040413</th>\n",
       "      <td>NOC</td>\n",
       "      <td>040413</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      SAB        CODE  value:float  lowerbound:float  \\\n",
       "CodeID:ID                                                              \n",
       "MTHSPL:J7A92W69L7  MTHSPL  J7A92W69L7          NaN               NaN   \n",
       "NCI:C76777            NCI      C76777          NaN               NaN   \n",
       "ATC:N07XX07           ATC     N07XX07          NaN               NaN   \n",
       "GS:1946                GS        1946          NaN               NaN   \n",
       "NOC:040413            NOC      040413          NaN               NaN   \n",
       "\n",
       "                   upperbound:float  unit  \n",
       "CodeID:ID                                  \n",
       "MTHSPL:J7A92W69L7               NaN   NaN  \n",
       "NCI:C76777                      NaN   NaN  \n",
       "ATC:N07XX07                     NaN   NaN  \n",
       "GS:1946                         NaN   NaN  \n",
       "NOC:040413                      NaN   NaN  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "codes = pd.read_csv(base_input + \"neo4j/import/CODEs.csv\", index_col=0)\n",
    "codes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:END_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000163</td>\n",
       "      <td>S0007806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000427</td>\n",
       "      <td>S0008336</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>S11922611</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000992</td>\n",
       "      <td>S1577859</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0001021</td>\n",
       "      <td>S0797872</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :START_ID    :END_ID\n",
       "0  C0000163   S0007806\n",
       "1  C0000427   S0008336\n",
       "2  C0000665  S11922611\n",
       "3  C0000992   S1577859\n",
       "4  C0001021   S0797872"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term = pd.read_csv(base_input + \"neo4j/import/CUI-SUIs.csv\")\n",
    "concept_term.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:END_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000132</td>\n",
       "      <td>T126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000246</td>\n",
       "      <td>T116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000895</td>\n",
       "      <td>T060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000908</td>\n",
       "      <td>T037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000931</td>\n",
       "      <td>T067</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :START_ID :END_ID\n",
       "0  C0000132    T126\n",
       "1  C0000246    T116\n",
       "2  C0000895    T060\n",
       "3  C0000908    T037\n",
       "4  C0000931    T067"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_semantics = pd.read_csv(base_input + \"neo4j/import/CUI-TUIs.csv\")\n",
    "concept_semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:END_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000294</td>\n",
       "      <td>ATC:V03AF01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000481</td>\n",
       "      <td>CHV:0000000513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000661</td>\n",
       "      <td>MSH:D015124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>VANDF:4020847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000737</td>\n",
       "      <td>LNC:LA15468-4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :START_ID         :END_ID\n",
       "0  C0000294     ATC:V03AF01\n",
       "1  C0000481  CHV:0000000513\n",
       "2  C0000661     MSH:D015124\n",
       "3  C0000665   VANDF:4020847\n",
       "4  C0000737   LNC:LA15468-4"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code = pd.read_csv(base_input + \"neo4j/import/CUI-CODEs.csv\")\n",
    "concept_code.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:END_ID</th>\n",
       "      <th>:START_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>T204</td>\n",
       "      <td>T002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>T001</td>\n",
       "      <td>T004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>T071</td>\n",
       "      <td>T004</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>T072</td>\n",
       "      <td>T010</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>T204</td>\n",
       "      <td>T010</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :END_ID :START_ID\n",
       "0    T204      T002\n",
       "1    T001      T004\n",
       "2    T071      T004\n",
       "3    T072      T010\n",
       "4    T204      T010"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "semantics_semantics = pd.read_csv(base_input + \"neo4j/import/TUIrel.csv\")\n",
    "semantics_semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:END_ID</th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:TYPE</th>\n",
       "      <th>CUI</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>S1424701</td>\n",
       "      <td>RXNORM:74</td>\n",
       "      <td>IN</td>\n",
       "      <td>C0000473</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>S18541041</td>\n",
       "      <td>SNOMEDCT_US:80994002</td>\n",
       "      <td>FN</td>\n",
       "      <td>C0000477</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>S11730064</td>\n",
       "      <td>SNOMEDCT_US:226367006</td>\n",
       "      <td>SY</td>\n",
       "      <td>C0000545</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>S0288461</td>\n",
       "      <td>CSP:2005-4146</td>\n",
       "      <td>PT</td>\n",
       "      <td>C0000735</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>S1957040</td>\n",
       "      <td>MDR:10048885</td>\n",
       "      <td>LLT</td>\n",
       "      <td>C0000735</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     :END_ID              :START_ID :TYPE       CUI\n",
       "0   S1424701              RXNORM:74    IN  C0000473\n",
       "1  S18541041   SNOMEDCT_US:80994002    FN  C0000477\n",
       "2  S11730064  SNOMEDCT_US:226367006    SY  C0000545\n",
       "3   S0288461          CSP:2005-4146    PT  C0000735\n",
       "4   S1957040           MDR:10048885   LLT  C0000735"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "code_term = pd.read_csv(base_input + \"neo4j/import/CODE-SUIs.csv\")\n",
    "code_term.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Merge Concept and Terms"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7923747, 2)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term.columns = [\"CUI:ID\", \"SUI:ID\"]\n",
    "concept_term.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SUI:ID</th>\n",
       "      <th>name</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CUI:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES1JP4KZ1 CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...</td>\n",
       "      <td>in situ Hi-C on HCT116 cells (containing AID-t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES21D8SP8 CUI</th>\n",
       "      <td>TWljcm8tQyBvbiBIMS1FU0MgY2VsbHMuSDEtRVND</td>\n",
       "      <td>Micro-C on H1-ESC cells.H1-ESC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES2M5JIGV CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uICBIaS1FU0MgY2VsbHMuSDEtRVND</td>\n",
       "      <td>in situ Hi-C on  Hi-ESC cells.H1-ESC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES2R6PUEK CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uIEhGRmM2IGNlbGxzLkhGRmM2</td>\n",
       "      <td>in situ Hi-C on HFFc6 cells.HFFc6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES3QAGOZZ CUI</th>\n",
       "      <td>aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...</td>\n",
       "      <td>in situ Hi-C on HCT116 cells (containing AID-t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                  SUI:ID  \\\n",
       "CUI:ID                                                                     \n",
       "4DND:4DNES1JP4KZ1 CUI  aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...   \n",
       "4DND:4DNES21D8SP8 CUI           TWljcm8tQyBvbiBIMS1FU0MgY2VsbHMuSDEtRVND   \n",
       "4DND:4DNES2M5JIGV CUI   aW4gc2l0dSBIaS1DIG9uICBIaS1FU0MgY2VsbHMuSDEtRVND   \n",
       "4DND:4DNES2R6PUEK CUI       aW4gc2l0dSBIaS1DIG9uIEhGRmM2IGNlbGxzLkhGRmM2   \n",
       "4DND:4DNES3QAGOZZ CUI  aW4gc2l0dSBIaS1DIG9uIEhDVDExNiBjZWxscyAoY29udG...   \n",
       "\n",
       "                                                                    name  \n",
       "CUI:ID                                                                    \n",
       "4DND:4DNES1JP4KZ1 CUI  in situ Hi-C on HCT116 cells (containing AID-t...  \n",
       "4DND:4DNES21D8SP8 CUI                     Micro-C on H1-ESC cells.H1-ESC  \n",
       "4DND:4DNES2M5JIGV CUI               in situ Hi-C on  Hi-ESC cells.H1-ESC  \n",
       "4DND:4DNES2R6PUEK CUI                  in situ Hi-C on HFFc6 cells.HFFc6  \n",
       "4DND:4DNES3QAGOZZ CUI  in situ Hi-C on HCT116 cells (containing AID-t...  "
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term = pd.merge(concept_term, terms, on=\"SUI:ID\", how='outer')\n",
    "concept_term = concept_term.groupby('CUI:ID').first()\n",
    "concept_term.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7923727, 2)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CUI:ID</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES1JP4KZ1 CUI</th>\n",
       "      <td>in situ Hi-C on HCT116 cells (containing AID-t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES21D8SP8 CUI</th>\n",
       "      <td>Micro-C on H1-ESC cells.H1-ESC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES2M5JIGV CUI</th>\n",
       "      <td>in situ Hi-C on  Hi-ESC cells.H1-ESC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES2R6PUEK CUI</th>\n",
       "      <td>in situ Hi-C on HFFc6 cells.HFFc6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DND:4DNES3QAGOZZ CUI</th>\n",
       "      <td>in situ Hi-C on HCT116 cells (containing AID-t...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                   label\n",
       "CUI:ID                                                                  \n",
       "4DND:4DNES1JP4KZ1 CUI  in situ Hi-C on HCT116 cells (containing AID-t...\n",
       "4DND:4DNES21D8SP8 CUI                     Micro-C on H1-ESC cells.H1-ESC\n",
       "4DND:4DNES2M5JIGV CUI               in situ Hi-C on  Hi-ESC cells.H1-ESC\n",
       "4DND:4DNES2R6PUEK CUI                  in situ Hi-C on HFFc6 cells.HFFc6\n",
       "4DND:4DNES3QAGOZZ CUI  in situ Hi-C on HCT116 cells (containing AID-t..."
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term.columns = [\"SUI:ID\", \"label\"]\n",
    "concept_term = concept_term[[\"label\"]]\n",
    "concept_term.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(7923727, 1)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_term.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "      <td>1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "      <td>3',5'-Cyclic-Nucleotide Phosphodiesterase</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "      <td>6-Aminonicotinamide</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "      <td>Abdominal Muscles</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "      <td>Academic Problem</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 label\n",
       "id                                                    \n",
       "C0000097  1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine\n",
       "C0000359     3',5'-Cyclic-Nucleotide Phosphodiesterase\n",
       "C0000610                           6-Aminonicotinamide\n",
       "C0000739                             Abdominal Muscles\n",
       "C0000873                              Academic Problem"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.loc[concept_term.index, 'label'] = concept_term.loc[concept_term.index, 'label']\n",
    "concepts.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Semantics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>name</th>\n",
       "      <th>STN</th>\n",
       "      <th>DEF</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>TUI:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>T012</th>\n",
       "      <td>Bird</td>\n",
       "      <td>A1.1.3.1.1.2</td>\n",
       "      <td>A vertebrate having a constant body temperatur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T014</th>\n",
       "      <td>Reptile</td>\n",
       "      <td>A1.1.3.1.1.5</td>\n",
       "      <td>A cold-blooded vertebrate having an external c...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T015</th>\n",
       "      <td>Mammal</td>\n",
       "      <td>A1.1.3.1.1.4</td>\n",
       "      <td>A vertebrate having a constant body temperatur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T021</th>\n",
       "      <td>Fully Formed Anatomical Structure</td>\n",
       "      <td>A1.2.3</td>\n",
       "      <td>An anatomical structure in a fully formed orga...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>T022</th>\n",
       "      <td>Body System</td>\n",
       "      <td>A2.1.4.1</td>\n",
       "      <td>A complex of anatomical structures that perfor...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                     name           STN  \\\n",
       "TUI:ID                                                    \n",
       "T012                                 Bird  A1.1.3.1.1.2   \n",
       "T014                              Reptile  A1.1.3.1.1.5   \n",
       "T015                               Mammal  A1.1.3.1.1.4   \n",
       "T021    Fully Formed Anatomical Structure        A1.2.3   \n",
       "T022                          Body System      A2.1.4.1   \n",
       "\n",
       "                                                      DEF  \n",
       "TUI:ID                                                     \n",
       "T012    A vertebrate having a constant body temperatur...  \n",
       "T014    A cold-blooded vertebrate having an external c...  \n",
       "T015    A vertebrate having a constant body temperatur...  \n",
       "T021    An anatomical structure in a fully formed orga...  \n",
       "T022    A complex of anatomical structures that perfor...  "
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>:START_ID</th>\n",
       "      <th>:END_ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000132</td>\n",
       "      <td>T126</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000246</td>\n",
       "      <td>T116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000895</td>\n",
       "      <td>T060</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000908</td>\n",
       "      <td>T037</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000931</td>\n",
       "      <td>T067</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  :START_ID :END_ID\n",
       "0  C0000132    T126\n",
       "1  C0000246    T116\n",
       "2  C0000895    T060\n",
       "3  C0000908    T037\n",
       "4  C0000931    T067"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "12264238"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "no_type = set(concepts.index) - set(concept_semantics[':START_ID'])\n",
    "len(no_type)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('out/0623/semantics_ranked.tsv') as o:\n",
    "    ranked_type = [i.strip() for i in o.read().strip().split(\"\\n\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>TUI:ID</th>\n",
       "      <th>type</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000132</td>\n",
       "      <td>T126</td>\n",
       "      <td>Enzyme</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000246</td>\n",
       "      <td>T116</td>\n",
       "      <td>Amino Acid, Peptide, or Protein</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000895</td>\n",
       "      <td>T060</td>\n",
       "      <td>Diagnostic Procedure</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000908</td>\n",
       "      <td>T037</td>\n",
       "      <td>Injury or Poisoning</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000931</td>\n",
       "      <td>T067</td>\n",
       "      <td>Phenomenon or Process</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id TUI:ID                             type\n",
       "0  C0000132   T126                           Enzyme\n",
       "1  C0000246   T116  Amino Acid, Peptide, or Protein\n",
       "2  C0000895   T060             Diagnostic Procedure\n",
       "3  C0000908   T037              Injury or Poisoning\n",
       "4  C0000931   T067            Phenomenon or Process"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_semantics.columns = [\"id\", \"TUI:ID\"]\n",
    "concept_semantics[\"type\"] = [semantics.at[i, 'name'] for i in concept_semantics['TUI:ID']]\n",
    "concept_semantics.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fetch_type(v):\n",
    "    cat = \"\"\n",
    "    rank = len(ranked_type)\n",
    "    for i in v:\n",
    "        r = ranked_type.index(i)\n",
    "        if r < rank:\n",
    "            cat = i\n",
    "            rank = r\n",
    "    return cat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "C0000005    Amino Acid, Peptide, or Protein; Indicator, Re...\n",
       "C0000039            Organic Chemical; Pharmacologic Substance\n",
       "C0000052              Amino Acid, Peptide, or Protein; Enzyme\n",
       "C0000074                                     Organic Chemical\n",
       "C0000084    Amino Acid, Peptide, or Protein; Biologically ...\n",
       "Name: type, dtype: object"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs = concept_semantics.groupby('id')['type'].apply(lambda x: \"; \".join(set(x)))\n",
    "cs.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "C0000005    Amino Acid, Peptide, or Protein\n",
       "C0000039                   Organic Chemical\n",
       "C0000052                             Enzyme\n",
       "C0000074                   Organic Chemical\n",
       "C0000084    Amino Acid, Peptide, or Protein\n",
       "Name: type, dtype: object"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cs_ranked = concept_semantics.groupby('id')['type'].apply(fetch_type)\n",
    "cs_ranked.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "id\n",
       "C4596166                                       Fungus\n",
       "C2131805                                      Finding\n",
       "C1550825                             Population Group\n",
       "C3884354    Organic Chemical; Pharmacologic Substance\n",
       "C0857614                                      Finding\n",
       "Name: type, dtype: object"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "common = list(set(concepts.index).intersection(cs.index))\n",
    "cs[common].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>type_combined</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "      <td>1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine</td>\n",
       "      <td>Organic Chemical</td>\n",
       "      <td>Organic Chemical; Hazardous or Poisonous Subst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "      <td>3',5'-Cyclic-Nucleotide Phosphodiesterase</td>\n",
       "      <td>Enzyme</td>\n",
       "      <td>Amino Acid, Peptide, or Protein; Enzyme</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "      <td>6-Aminonicotinamide</td>\n",
       "      <td>Pharmacologic Substance</td>\n",
       "      <td>Pharmacologic Substance; Vitamin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "      <td>Abdominal Muscles</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "      <td>Academic Problem</td>\n",
       "      <td>Finding</td>\n",
       "      <td>Finding</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 label  \\\n",
       "id                                                       \n",
       "C0000097  1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine   \n",
       "C0000359     3',5'-Cyclic-Nucleotide Phosphodiesterase   \n",
       "C0000610                           6-Aminonicotinamide   \n",
       "C0000739                             Abdominal Muscles   \n",
       "C0000873                              Academic Problem   \n",
       "\n",
       "                                          type  \\\n",
       "id                                               \n",
       "C0000097                      Organic Chemical   \n",
       "C0000359                                Enzyme   \n",
       "C0000610               Pharmacologic Substance   \n",
       "C0000739  Body Part, Organ, or Organ Component   \n",
       "C0000873                               Finding   \n",
       "\n",
       "                                              type_combined  \n",
       "id                                                           \n",
       "C0000097  Organic Chemical; Hazardous or Poisonous Subst...  \n",
       "C0000359            Amino Acid, Peptide, or Protein; Enzyme  \n",
       "C0000610                   Pharmacologic Substance; Vitamin  \n",
       "C0000739               Body Part, Organ, or Organ Component  \n",
       "C0000873                                            Finding  "
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_semantics\n",
    "concepts.loc[common, 'type'] = cs_ranked[common]\n",
    "concepts.loc[common, 'type_combined'] = cs[common]\n",
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "out_prefix = \"out/0915/\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "concepts.groupby(\"type_combined\").first().to_csv(out_prefix + 'semantics.tsv', sep=\"\\t\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>type_combined</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "      <td>1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine</td>\n",
       "      <td>Organic Chemical</td>\n",
       "      <td>Organic Chemical; Hazardous or Poisonous Subst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "      <td>3',5'-Cyclic-Nucleotide Phosphodiesterase</td>\n",
       "      <td>Enzyme</td>\n",
       "      <td>Amino Acid, Peptide, or Protein; Enzyme</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "      <td>6-Aminonicotinamide</td>\n",
       "      <td>Pharmacologic Substance</td>\n",
       "      <td>Pharmacologic Substance; Vitamin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "      <td>Abdominal Muscles</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "      <td>Academic Problem</td>\n",
       "      <td>Finding</td>\n",
       "      <td>Finding</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 label  \\\n",
       "id                                                       \n",
       "C0000097  1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine   \n",
       "C0000359     3',5'-Cyclic-Nucleotide Phosphodiesterase   \n",
       "C0000610                           6-Aminonicotinamide   \n",
       "C0000739                             Abdominal Muscles   \n",
       "C0000873                              Academic Problem   \n",
       "\n",
       "                                          type  \\\n",
       "id                                               \n",
       "C0000097                      Organic Chemical   \n",
       "C0000359                                Enzyme   \n",
       "C0000610               Pharmacologic Substance   \n",
       "C0000739  Body Part, Organ, or Organ Component   \n",
       "C0000873                               Finding   \n",
       "\n",
       "                                              type_combined  \n",
       "id                                                           \n",
       "C0000097  Organic Chemical; Hazardous or Poisonous Subst...  \n",
       "C0000359            Amino Acid, Peptide, or Protein; Enzyme  \n",
       "C0000610                   Pharmacologic Substance; Vitamin  \n",
       "C0000739               Body Part, Organ, or Organ Component  \n",
       "C0000873                                            Finding  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(15527671, 3)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(out_prefix + 'semantics_list.tsv', 'w') as o:\n",
    "    o.write(\"\\n\".join([str(i) for i in concept_semantics.type.unique()]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SAB</th>\n",
       "      <th>CODE</th>\n",
       "      <th>value:float</th>\n",
       "      <th>lowerbound:float</th>\n",
       "      <th>upperbound:float</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CodeID:ID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MTHSPL:J7A92W69L7</th>\n",
       "      <td>MTHSPL</td>\n",
       "      <td>J7A92W69L7</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NCI:C76777</th>\n",
       "      <td>NCI</td>\n",
       "      <td>C76777</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ATC:N07XX07</th>\n",
       "      <td>ATC</td>\n",
       "      <td>N07XX07</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GS:1946</th>\n",
       "      <td>GS</td>\n",
       "      <td>1946</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>NOC:040413</th>\n",
       "      <td>NOC</td>\n",
       "      <td>040413</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      SAB        CODE  value:float  lowerbound:float  \\\n",
       "CodeID:ID                                                              \n",
       "MTHSPL:J7A92W69L7  MTHSPL  J7A92W69L7          NaN               NaN   \n",
       "NCI:C76777            NCI      C76777          NaN               NaN   \n",
       "ATC:N07XX07           ATC     N07XX07          NaN               NaN   \n",
       "GS:1946                GS        1946          NaN               NaN   \n",
       "NOC:040413            NOC      040413          NaN               NaN   \n",
       "\n",
       "                   upperbound:float  unit  \n",
       "CodeID:ID                                  \n",
       "MTHSPL:J7A92W69L7               NaN   NaN  \n",
       "NCI:C76777                      NaN   NaN  \n",
       "ATC:N07XX07                     NaN   NaN  \n",
       "GS:1946                         NaN   NaN  \n",
       "NOC:040413                      NaN   NaN  "
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "codes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>CodeID:ID</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000294</td>\n",
       "      <td>ATC:V03AF01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000481</td>\n",
       "      <td>CHV:0000000513</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000661</td>\n",
       "      <td>MSH:D015124</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>VANDF:4020847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000737</td>\n",
       "      <td>LNC:LA15468-4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id       CodeID:ID\n",
       "0  C0000294     ATC:V03AF01\n",
       "1  C0000481  CHV:0000000513\n",
       "2  C0000661     MSH:D015124\n",
       "3  C0000665   VANDF:4020847\n",
       "4  C0000737   LNC:LA15468-4"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code.columns = [\"id\", \"CodeID:ID\"]\n",
    "concept_code.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>CodeID:ID</th>\n",
       "      <th>SAB</th>\n",
       "      <th>CODE</th>\n",
       "      <th>value:float</th>\n",
       "      <th>lowerbound:float</th>\n",
       "      <th>upperbound:float</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0000294</td>\n",
       "      <td>ATC:V03AF01</td>\n",
       "      <td>ATC</td>\n",
       "      <td>V03AF01</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0000481</td>\n",
       "      <td>CHV:0000000513</td>\n",
       "      <td>CHV</td>\n",
       "      <td>0000000513</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0000661</td>\n",
       "      <td>MSH:D015124</td>\n",
       "      <td>MSH</td>\n",
       "      <td>D015124</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0000665</td>\n",
       "      <td>VANDF:4020847</td>\n",
       "      <td>VANDF</td>\n",
       "      <td>4020847</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0000737</td>\n",
       "      <td>LNC:LA15468-4</td>\n",
       "      <td>LNC</td>\n",
       "      <td>LA15468-4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         id       CodeID:ID    SAB        CODE  value:float  lowerbound:float  \\\n",
       "0  C0000294     ATC:V03AF01    ATC     V03AF01          NaN               NaN   \n",
       "1  C0000481  CHV:0000000513    CHV  0000000513          NaN               NaN   \n",
       "2  C0000661     MSH:D015124    MSH     D015124          NaN               NaN   \n",
       "3  C0000665   VANDF:4020847  VANDF     4020847          NaN               NaN   \n",
       "4  C0000737   LNC:LA15468-4    LNC   LA15468-4          NaN               NaN   \n",
       "\n",
       "   upperbound:float  unit  \n",
       "0               NaN   NaN  \n",
       "1               NaN   NaN  \n",
       "2               NaN   NaN  \n",
       "3               NaN   NaN  \n",
       "4               NaN   NaN  "
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code = pd.merge(concept_code, codes, on=\"CodeID:ID\", how='left')\n",
    "concept_code.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>CodeID:ID</th>\n",
       "      <th>SAB</th>\n",
       "      <th>CODE</th>\n",
       "      <th>value:float</th>\n",
       "      <th>lowerbound:float</th>\n",
       "      <th>upperbound:float</th>\n",
       "      <th>unit</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>623690</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>LCH_NW:sh86002892</td>\n",
       "      <td>LCH_NW</td>\n",
       "      <td>sh86002892</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1108107</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>CSP:2511-0411</td>\n",
       "      <td>CSP</td>\n",
       "      <td>2511-0411</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1246274</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>PSY:31213</td>\n",
       "      <td>PSY</td>\n",
       "      <td>31213</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2179442</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>MSH:D015632</td>\n",
       "      <td>MSH</td>\n",
       "      <td>D015632</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3529094</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>CHV:0000000501</td>\n",
       "      <td>CHV</td>\n",
       "      <td>0000000501</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3736237</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>PSY:32433</td>\n",
       "      <td>PSY</td>\n",
       "      <td>32433</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4116966</th>\n",
       "      <td>C0000097</td>\n",
       "      <td>SNOMEDCT_US:285407008</td>\n",
       "      <td>SNOMEDCT_US</td>\n",
       "      <td>285407008</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               id              CodeID:ID          SAB        CODE  \\\n",
       "623690   C0000097      LCH_NW:sh86002892       LCH_NW  sh86002892   \n",
       "1108107  C0000097          CSP:2511-0411          CSP   2511-0411   \n",
       "1246274  C0000097              PSY:31213          PSY       31213   \n",
       "2179442  C0000097            MSH:D015632          MSH     D015632   \n",
       "3529094  C0000097         CHV:0000000501          CHV  0000000501   \n",
       "3736237  C0000097              PSY:32433          PSY       32433   \n",
       "4116966  C0000097  SNOMEDCT_US:285407008  SNOMEDCT_US   285407008   \n",
       "\n",
       "         value:float  lowerbound:float  upperbound:float  unit  \n",
       "623690           NaN               NaN               NaN   NaN  \n",
       "1108107          NaN               NaN               NaN   NaN  \n",
       "1246274          NaN               NaN               NaN   NaN  \n",
       "2179442          NaN               NaN               NaN   NaN  \n",
       "3529094          NaN               NaN               NaN   NaN  \n",
       "3736237          NaN               NaN               NaN   NaN  \n",
       "4116966          NaN               NaN               NaN   NaN  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concept_code[concept_code.id == 'C0000097']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>type_combined</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C0000097</th>\n",
       "      <td>1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine</td>\n",
       "      <td>Organic Chemical</td>\n",
       "      <td>Organic Chemical; Hazardous or Poisonous Subst...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000359</th>\n",
       "      <td>3',5'-Cyclic-Nucleotide Phosphodiesterase</td>\n",
       "      <td>Enzyme</td>\n",
       "      <td>Amino Acid, Peptide, or Protein; Enzyme</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000610</th>\n",
       "      <td>6-Aminonicotinamide</td>\n",
       "      <td>Pharmacologic Substance</td>\n",
       "      <td>Pharmacologic Substance; Vitamin</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000739</th>\n",
       "      <td>Abdominal Muscles</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "      <td>Body Part, Organ, or Organ Component</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C0000873</th>\n",
       "      <td>Academic Problem</td>\n",
       "      <td>Finding</td>\n",
       "      <td>Finding</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 label  \\\n",
       "id                                                       \n",
       "C0000097  1-Methyl-4-phenyl-1,2,3,6-tetrahydropyridine   \n",
       "C0000359     3',5'-Cyclic-Nucleotide Phosphodiesterase   \n",
       "C0000610                           6-Aminonicotinamide   \n",
       "C0000739                             Abdominal Muscles   \n",
       "C0000873                              Academic Problem   \n",
       "\n",
       "                                          type  \\\n",
       "id                                               \n",
       "C0000097                      Organic Chemical   \n",
       "C0000359                                Enzyme   \n",
       "C0000610               Pharmacologic Substance   \n",
       "C0000739  Body Part, Organ, or Organ Component   \n",
       "C0000873                               Finding   \n",
       "\n",
       "                                              type_combined  \n",
       "id                                                           \n",
       "C0000097  Organic Chemical; Hazardous or Poisonous Subst...  \n",
       "C0000359            Amino Acid, Peptide, or Protein; Enzyme  \n",
       "C0000610                   Pharmacologic Substance; Vitamin  \n",
       "C0000739               Body Part, Organ, or Organ Component  \n",
       "C0000873                                            Finding  "
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "type_mapper = {}\n",
    "with open(\"output/unique_SABS_of_Concept_Mapper.txt\") as o:\n",
    "    for line in o:\n",
    "        r = line.strip().split(\":\")\n",
    "        if len(r) == 2:\n",
    "            type_mapper[r[0]] = r[1]\n",
    "        elif 'MSIGDB' in r[0]:\n",
    "            type_mapper[r[0]] = 'MSIGDB'\n",
    "        else:\n",
    "            type_mapper[r[0]] = r[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "15721067it [07:40, 34108.59it/s]\n"
     ]
    }
   ],
   "source": [
    "for i,row in tqdm(concept_code[concept_code.id.isin(concepts[concepts.type.isna()].index)].iterrows()):\n",
    "    sab = row[\"SAB\"]\n",
    "    ind = row[\"id\"]\n",
    "    if type(sab) == str:\n",
    "        if 'MSIGDB' in sab:\n",
    "            sab = 'MSIGDB'\n",
    "            concept_code.at[i, 'SAB'] = 'MSIGDB'\n",
    "        if sab == 'MSIGDB':\n",
    "            tp = 'MSIGDB'\n",
    "        else:\n",
    "            tp = type_mapper[sab]\n",
    "        if tp:\n",
    "            concepts.at[ind, \"type\"] = tp\n",
    "            concepts.at[ind, \"type_combined\"] = tp\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in concepts.iterrows():\n",
    "    concepts.at[i, \"type\"] = row[\"type\"].replace(\".\", \" \")\n",
    "    concepts.at[i, \"type_combined\"] = row[\"type_combined\"].replace(\".\", \" \")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>type_combined</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:A0A087X1C5 CUI</th>\n",
       "      <td>CP2D7_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:A0A0B4J2F0 CUI</th>\n",
       "      <td>PIOS1_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:A0A0B4J2F2 CUI</th>\n",
       "      <td>SIK1B_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:A0A0C5B5G6 CUI</th>\n",
       "      <td>MOTSC_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:A0A0K2S4Q6 CUI</th>\n",
       "      <td>CD3CH_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                label       type type_combined\n",
       "id                                                            \n",
       "UNIPROTKB:A0A087X1C5 CUI  CP2D7_HUMAN  UNIPROTKB     UNIPROTKB\n",
       "UNIPROTKB:A0A0B4J2F0 CUI  PIOS1_HUMAN  UNIPROTKB     UNIPROTKB\n",
       "UNIPROTKB:A0A0B4J2F2 CUI  SIK1B_HUMAN  UNIPROTKB     UNIPROTKB\n",
       "UNIPROTKB:A0A0C5B5G6 CUI  MOTSC_HUMAN  UNIPROTKB     UNIPROTKB\n",
       "UNIPROTKB:A0A0K2S4Q6 CUI  CD3CH_HUMAN  UNIPROTKB     UNIPROTKB"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "concepts[concepts.type == \"UNIPROTKB\"].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for tp in tqdm(concepts.type.unique()):\n",
    "    con = concepts[concepts.type==tp].copy()\n",
    "    cc = concept_code[concept_code.id.isin(con.index)]\n",
    "    for sab in cc.SAB.unique():\n",
    "        c = cc[cc.SAB == sab]\n",
    "        c = c.groupby('id').first()\n",
    "        common = list(set(con.index).intersection(c.index))\n",
    "        con.loc[common, sab] = c.loc[common, \"CodeID:ID\"]\n",
    "        if c.loc[common, \"value:float\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s value\"%sab] = c.loc[common, \"value:float\"]\n",
    "        if c.loc[common, \"lowerbound:float\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s lowerbound\"%sab] = c.loc[common, \"lowerbound:float\"]\n",
    "        if c.loc[common, \"upperbound:float\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s upperbound\"%sab] = c.loc[common, \"upperbound:float\"]\n",
    "        if c.loc[common, \"unit\"].isna().sum() != len(common):\n",
    "            con.loc[common, \"%s unit\"%sab] = c.loc[common, \"unit\"]\n",
    "        if \"-\" in list(con[\"label\"]):\n",
    "            tmp = con[con.label == \"-\"]\n",
    "            ind = set(tmp.index).intersection(con.index)\n",
    "            ind2 = set(tmp.index).intersection(c.index)\n",
    "            if len(ind.intersection(ind2)) > 0:\n",
    "                l = list(ind.intersection(ind2))\n",
    "                con.loc[l, \"label\"] = c.loc[l, 'CodeID:ID']\n",
    "        con.to_csv(\"out/0915/serialization/nodes/%s.nodes.csv\"%(tp))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_11529/346744825.py:3: DtypeWarning: Columns (15,16,19,27) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  gene_df = pd.read_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\", index_col=0)\n"
     ]
    }
   ],
   "source": [
    "gene_or_genome_df = pd.read_csv(\"out/0915/serialization/nodes/Gene or Genome.nodes.csv\", index_col=0)\n",
    "uniprot = pd.read_csv(\"out/0915/serialization/nodes/UNIPROTKB.nodes.csv\", index_col=0)\n",
    "gene_df = pd.read_csv(\"out/0915/serialization/nodes/Gene.nodes.csv\", index_col=0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:Q92993 CUI</th>\n",
       "      <td>KAT5_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:Q92993</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:Q9H093 CUI</th>\n",
       "      <td>NUAK2_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:Q9H093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:Q15418 CUI</th>\n",
       "      <td>KS6A1_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:Q15418</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P59540 CUI</th>\n",
       "      <td>T2R46_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:P59540</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P29122 CUI</th>\n",
       "      <td>PCSK6_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:P29122</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            label       type         UNIPROTKB\n",
       "id                                                            \n",
       "UNIPROTKB:Q92993 CUI   KAT5_HUMAN  UNIPROTKB  UNIPROTKB:Q92993\n",
       "UNIPROTKB:Q9H093 CUI  NUAK2_HUMAN  UNIPROTKB  UNIPROTKB:Q9H093\n",
       "UNIPROTKB:Q15418 CUI  KS6A1_HUMAN  UNIPROTKB  UNIPROTKB:Q15418\n",
       "UNIPROTKB:P59540 CUI  T2R46_HUMAN  UNIPROTKB  UNIPROTKB:P59540\n",
       "UNIPROTKB:P29122 CUI  PCSK6_HUMAN  UNIPROTKB  UNIPROTKB:P29122"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniprot.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((2546, 3), (39920, 28), (1, 5))"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniprot.shape, gene_df.shape, gene_or_genome_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>MTH</th>\n",
       "      <th>PUBCHEM</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C2828054</th>\n",
       "      <td>FANCD2 wt Allele</td>\n",
       "      <td>Gene or Genome</td>\n",
       "      <td>NCI:C86550</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>PUBCHEM:643975</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     label            type         NCI         MTH  \\\n",
       "id                                                                   \n",
       "C2828054  FANCD2 wt Allele  Gene or Genome  NCI:C86550  MTH:NOCODE   \n",
       "\n",
       "                 PUBCHEM  \n",
       "id                        \n",
       "C2828054  PUBCHEM:643975  "
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_or_genome_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniprot_id_mapper = pd.read_csv('output/idmapping_2023_09_18.tsv', sep=\"\\t\", index_col=0)\n",
    "uniprot_id_mapper.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_gene_or_genome = gene_or_genome_df[gene_or_genome_df.HGNC.isna()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "rows = {}\n",
    "hgnc_mapper = {}\n",
    "for i, row in gene_or_genome_df[~gene_or_genome_df.HGNC.isna()].iterrows():\n",
    "    hgnc = row[\"HGNC\"]\n",
    "    hgnc_mapper[hgnc] = i\n",
    "    row[\"type\"] = \"Gene\"\n",
    "    row[\"type_combined\"] = row[\"type_combined\"].replace(\"Gene or Genome\", \"Gene\")\n",
    "    rows[i] = row\n",
    "len(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in gene_df.iterrows():\n",
    "    hgnc = row[\"HGNC\"]\n",
    "    if hgnc not in hgnc_mapper:\n",
    "        hgnc_mapper[hgnc] = i\n",
    "        row[\"type\"] = \"Gene\"\n",
    "        rows[i] = row\n",
    "len(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniprot.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniprot_kb_mapper = {}\n",
    "uniprot_list = []\n",
    "with open(\"uniprot_ids_0917.txt\", \"w\") as o:\n",
    "    for i, row in uniprot.iterrows():\n",
    "        kb = row[\"UNIPROTKB\"].replace(\"UNIPROTKB:\", \"\")\n",
    "        o.write(\"%s\\n\"%kb)\n",
    "    # hgnc = uniprot_id_mapper.at[kb, 'To']\n",
    "    # hgnc = row[\"HGNC\"]\n",
    "    # if hgnc not in hgnc_mapper:\n",
    "    #     hgnc_mapper[hgnc] = i\n",
    "    #     row[\"type\"] = \"Gene\"\n",
    "    #     rows[i] = row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "uniprot_mapper = {}\n",
    "for k, v in uniprot_id_mapper.iterrows():\n",
    "    uniprot_mapper[k] = v[\"To\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "no_hgnc = set()\n",
    "for i, row in uniprot.iterrows():\n",
    "    kb = row[\"UNIPROTKB\"].replace(\"UNIPROTKB:\", \"\")\n",
    "    if kb not in uniprot_mapper:\n",
    "        no_hgnc.add(kb)\n",
    "        rows[i] = row\n",
    "    else:\n",
    "        hgnc = uniprot_mapper[kb]\n",
    "        if hgnc in hgnc_mapper:\n",
    "            cui = hgnc_mapper[hgnc]\n",
    "            rows[cui][\"UNIPROTKB\"] = kb\n",
    "        else:\n",
    "            row[\"HGNC\"] = hgnc\n",
    "            row[\"type\"] = \"Gene\"\n",
    "            row[\"type_combined\"] = \"Gene\"\n",
    "            rows[i] = row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_gene_df = pd.DataFrame.from_dict(rows, orient=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_gene_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_gene_df.type_combined = \"Gene\"\n",
    "new_gene_df.type = \"Gene\"\n",
    "new_gene_df.type_combined.unique(), new_gene_df.type.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "concepts.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in new_gene_df.index:\n",
    "    concepts.at[i, \"type\"] = \"Gene\"\n",
    "    concepts.at[i, \"type_combined\"] = \"Gene\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_gene_df.to_csv(\"out/0915/serialization/nodes/Gene.nodes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "new_gene_or_genome.to_csv(\"out/0915/serialization/nodes/Gene or Genome.nodes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "row_headers = [\"source\", \"relation\", \"target\", \"source_label\", \"target_label\", \"SAB\", \"evidence\"]\n",
    "with open(base_input + \"neo4j/import/CUI-CUIs.csv\") as o:\n",
    "    csv_reader = csv.reader(o)\n",
    "    headers = None\n",
    "    for row in tqdm(csv_reader):\n",
    "        if not headers:\n",
    "            headers = row\n",
    "        else:\n",
    "            source = row[0]\n",
    "            if source in uniprot_mapper:\n",
    "                source = uniprot_mapper[source]\n",
    "            target = row[1]\n",
    "            if target in uniprot_mapper:\n",
    "                target = uniprot_mapper[target]\n",
    "            if source in concepts.index and target in concepts.index:\n",
    "                source_label = concepts.at[source, 'label']\n",
    "                source_type = concepts.at[source, 'type']\n",
    "                \n",
    "                target_label = concepts.at[target, 'label']\n",
    "                target_type = concepts.at[target, 'type']\n",
    "                relation = row[2]\n",
    "                sab = row[3]\n",
    "                evidence = ''\n",
    "                if len(row) > 4:\n",
    "                    evidence = row[4]\n",
    "                filename = 'out/0915/serialization/edges/%s.%s.%s.edges.csv'%(source_type, relation, target_type)\n",
    "                write_header = False\n",
    "                operation = \"a\"\n",
    "                if not os.path.isfile(filename):\n",
    "                    write_header = True\n",
    "                    operation = \"w\"\n",
    "                # source_list = set()\n",
    "                # target_list = set()\n",
    "                with open(filename, operation) as w:\n",
    "                    csv_writer = csv.writer(w)\n",
    "                    if write_header:\n",
    "                        csv_writer.writerow(row_headers)\n",
    "                    csv_writer.writerow([source, relation, target, source_label, target_label, sab, evidence])\n",
    "                #     source_list.add(source)\n",
    "                #     target_list.add(target)\n",
    "\n",
    "                # # take note of nodes that are used for source and target\n",
    "                # source_ids = \"out/serialization/ids/%s.txt\"%source_type\n",
    "                # if not os.path.isfile(source_ids):\n",
    "                #     with open(source_ids, 'w') as o:\n",
    "                #         o.write(\"\\n\".join(source_list))\n",
    "                # else:\n",
    "                #     with open(source_ids) as o:\n",
    "                #         source_list = source_list.union(o.read().strip().split(\"\\n\"))\n",
    "                #     with open(source_ids, 'w') as o:\n",
    "                #         o.write(\"\\n\".join(source_list))\n",
    "                # target_ids = \"out/serialization/ids/%s.txt\"%target_type\n",
    "                # if not os.path.isfile(target_ids):\n",
    "                #     with open(target_ids, 'w') as o:\n",
    "                #         o.write(\"\\n\".join(target_list))\n",
    "                # else:\n",
    "                #     with open(target_ids) as o:\n",
    "                #         target_list = target_list.union(o.read().strip().split(\"\\n\"))\n",
    "                #     with open(target_ids, 'w') as o:\n",
    "                #         o.write(\"\\n\".join(target_list))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "for filename in glob(\"out/0915/serialization/nodes/*.csv\"):\n",
    "    df = pd.read_csv(filename, index_col=0, low_memory=False)\n",
    "    orig_columns = df.columns\n",
    "    if \"type_combined\" in df.columns:\n",
    "        dtype = df.type.unique()[0]\n",
    "        combined = set()\n",
    "        for i in df.type_combined:\n",
    "            combined = combined.union(i.split(\"; \"))\n",
    "        # remove og type\n",
    "        combined = combined - {dtype}\n",
    "        columns = [i for i in df.columns if not i == \"type_combined\"] + list(combined)\n",
    "        if len(combined) > 0:\n",
    "            print(filename)\n",
    "            for i in combined:\n",
    "                df[i] = False\n",
    "            for i, row in df.iterrows():\n",
    "                type_combined = row[\"type_combined\"].split(\"; \")\n",
    "                for t in type_combined:\n",
    "                    col = \"is_%s\"%t\n",
    "                    df.at[i, col] = True\n",
    "        df = df[columns]\n",
    "        df.to_csv(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open(\"output/august_dcc_sabs.txt\") as o:\n",
    "    sabs_to_keep = set(o.read().strip().split(\"\\n\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re\n",
    "import os\n",
    "edge_pattern = \"(?P<directory>.+)/(?P<source_type>.+)\\.(?P<relation>.+)\\.(?P<target_type>.+)\\.(?P<entity>.+)\\.csv\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [],
   "source": [
    "node_base = \"out/0915/serialization/nodes/%s.nodes.csv\"\n",
    "new_node_base = \"out/0915/filtered/nodes/%s.nodes.csv\"\n",
    "new_edge_base = \"out/0915/filtered/edges/%s.%s.%s.edges.csv\"\n",
    "ids_base = \"out/0915/filtered/ids/%s.txt\"\n",
    "node_ids = {}\n",
    "sab_relations = {}\n",
    "processed = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [],
   "source": [
    "def glygen(s):\n",
    "    return s.replace(\"GLYGEN.RESIDUE\", \"GLYGEN_RESIDUE\").replace(\"GLYCAN.MOTIF\", \"GLYCAN_MOTIF\").replace('GLYCOSYLTRANSFERASE.REACTION', 'GLYCOSYLTRANSFERASE_REACTION').replace(\"GLYGEN.SRC\", \"GLYGEN_SRC\").replace('GLYGEN.GLYCOSYLATION', 'GLYGEN_GLYCOSYLATION')\n",
    "\n",
    "def glygen_reverse(s):\n",
    "    return s.replace(\"GLYGEN_RESIDUE\", \"GLYGEN.RESIDUE\").replace(\"GLYCAN_MOTIF\", \"GLYCAN.MOTIF\").replace('GLYCOSYLTRANSFERASE_REACTION', 'GLYCOSYLTRANSFERASE.REACTION').replace(\"GLYGEN_SRC\", \"GLYGEN.SRC\").replace('GLYGEN_GLYCOSYLATION', 'GLYGEN.GLYCOSYLATION')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 107,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 75759/75759 [06:31<00:00, 193.49it/s] \n"
     ]
    }
   ],
   "source": [
    "for filename in tqdm(glob(\"out/0915/serialization/edges/*.csv\")):\n",
    "    if filename not in processed:\n",
    "        match = re.match(edge_pattern, glygen(filename)).groupdict()\n",
    "        entity = match[\"entity\"]\n",
    "        source_type = glygen_reverse(match[\"source_type\"])\n",
    "        relation = match[\"relation\"].replace(\"_\", \" \")\n",
    "        target_type = glygen_reverse(match[\"target_type\"])\n",
    "        if \"inverse\" not in relation:\n",
    "            edge_df = pd.read_csv(filename, low_memory=False)\n",
    "            # filter for SAB\n",
    "            sabs = sabs_to_keep.intersection(edge_df.SAB.unique())\n",
    "            for sab in sabs:\n",
    "                if sab not in sab_relations:\n",
    "                    sab_relations[sab] = set()\n",
    "                sab_relations[sab].add(relation)\n",
    "            if len(sabs) > 0:\n",
    "                edge_df = edge_df[edge_df.SAB.isin(sabs)]\n",
    "                if not os.path.isfile(ids_base%source_type):\n",
    "                    with open(ids_base%source_type, 'w') as o:\n",
    "                        o.write(\"\\n\".join(edge_df.source))\n",
    "                else:\n",
    "                    with open(ids_base%source_type) as o:\n",
    "                        ids = set(o.read().strip().split(\"\\n\"))\n",
    "                    with open(ids_base%source_type, 'w') as o:\n",
    "                        ids = ids.union(edge_df.source)\n",
    "                        o.write(\"\\n\".join(ids))\n",
    "                if not os.path.isfile(ids_base%target_type):\n",
    "                    with open(ids_base%target_type, 'w') as o:\n",
    "                        o.write(\"\\n\".join(edge_df.target))\n",
    "                else:\n",
    "                    with open(ids_base%target_type) as o:\n",
    "                        ids = set(o.read().strip().split(\"\\n\"))\n",
    "                    with open(ids_base%target_type, 'w') as o:\n",
    "                        ids = ids.union(edge_df.target)\n",
    "                        o.write(\"\\n\".join(ids))\n",
    "                # source_df = pd.read_csv(node_base%source_type, index_col=0, low_memory=False)\n",
    "                # if os.path.isfile(new_node_base%(source_type)):\n",
    "                #     new_source_df = pd.read_csv(new_node_base%(source_type), index_col=0, low_memory=False)\n",
    "                #     pd.concat([new_source_df, source_df]).dropna(axis=1).to_csv(new_node_base%(source_type))\n",
    "                # else:\n",
    "                #     source_df.dropna(axis=1).to_csv(new_node_base%(source_type))\n",
    "                \n",
    "                # target_df = pd.read_csv(node_base%target_type, index_col=0, low_memory=False)\n",
    "                # if os.path.isfile(new_node_base%(target_type)):\n",
    "                #     new_target_df = pd.read_csv(new_node_base%(target_type), index_col=0, low_memory=False)\n",
    "                #     pd.concat([new_target_df, target_df]).dropna(axis=1).to_csv(new_node_base%(target_type))\n",
    "                # else:\n",
    "                #     target_df.dropna(axis=1).to_csv(new_node_base%(target_type))\n",
    "                edge_df.to_csv(new_edge_base%(source_type, relation, target_type), index=False)\n",
    "        processed.add(filename.replace(\"GLYGEN_RESIDUE\", \"GLYGEN.RESIDUE\"))\n",
    "                "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 117/117 [00:00<00:00, 2921033.14it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "117"
      ]
     },
     "execution_count": 108,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "count = 0\n",
    "for filename in tqdm(glob(\"out/0915/filtered/ids/*.txt\")):\n",
    "    count+=1\n",
    "count"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 117/117 [01:07<00:00,  1.73it/s]\n"
     ]
    }
   ],
   "source": [
    "id_pattern = \"(?P<directory>.+)/(?P<type>.+)\\.txt\"\n",
    "for filename in tqdm(glob(\"out/0915/filtered/ids/*.txt\")):\n",
    "    if not \"inverse\" in filename and not \"isa_\" in filename:\n",
    "        match = re.match(id_pattern, filename).groupdict()\n",
    "        node_type = match[\"type\"]\n",
    "        node_df = pd.read_csv(node_base%node_type, index_col=0, low_memory=False)\n",
    "        with open(filename) as o:\n",
    "            ids = list(set(o.read().strip().split(\"\\n\")).intersection(node_df.index))\n",
    "        node_df.loc[ids].dropna(axis=1, how=\"all\").to_csv(new_node_base%node_type)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "hgnc = pd.read_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\", low_memory=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Unnamed: 0</th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>LCH_NW</th>\n",
       "      <th>MSH</th>\n",
       "      <th>CSP</th>\n",
       "      <th>SNOMEDCT_US</th>\n",
       "      <th>HGNC</th>\n",
       "      <th>MTH</th>\n",
       "      <th>...</th>\n",
       "      <th>CHEBI</th>\n",
       "      <th>ENSEMBL</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "      <th>ENTREZ lowerbound</th>\n",
       "      <th>ENTREZ upperbound</th>\n",
       "      <th>PUBCHEM</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C1420882</td>\n",
       "      <td>TPT1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C101389</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12022</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000133112</td>\n",
       "      <td>17.0</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341284.0</td>\n",
       "      <td>ENTREZ:7178</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341183.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P13693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C1412662</td>\n",
       "      <td>ATP5MC1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:841</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000159199</td>\n",
       "      <td>14.0</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>ENTREZ:516</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P05496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C1824623</td>\n",
       "      <td>CCDC96 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:26900</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000173013</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>ENTREZ:257236</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q2M329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C3543374</td>\n",
       "      <td>RN7SL531P gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:46547</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000264071</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43492032.0</td>\n",
       "      <td>43492313.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C1336927</td>\n",
       "      <td>VAV3 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C24894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12659</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000134215</td>\n",
       "      <td>16.0</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>ENTREZ:10451</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9UKW4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  Unnamed: 0           label  type          NCI LCH_NW  MSH  CSP SNOMEDCT_US  \\\n",
       "0   C1420882       TPT1 gene  Gene  NCI:C101389    NaN  NaN  NaN         NaN   \n",
       "1   C1412662    ATP5MC1 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "2   C1824623     CCDC96 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "3   C3543374  RN7SL531P gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "4   C1336927       VAV3 gene  Gene   NCI:C24894    NaN  NaN  NaN         NaN   \n",
       "\n",
       "         HGNC         MTH  ... CHEBI                  ENSEMBL ENSEMBL value  \\\n",
       "0  HGNC:12022  MTH:NOCODE  ...   NaN  ENSEMBL:ENSG00000133112          17.0   \n",
       "1    HGNC:841  MTH:NOCODE  ...   NaN  ENSEMBL:ENSG00000159199          14.0   \n",
       "2  HGNC:26900  MTH:NOCODE  ...   NaN  ENSEMBL:ENSG00000173013           6.0   \n",
       "3  HGNC:46547  MTH:NOCODE  ...   NaN  ENSEMBL:ENSG00000264071           2.0   \n",
       "4  HGNC:12659  MTH:NOCODE  ...   NaN  ENSEMBL:ENSG00000134215          16.0   \n",
       "\n",
       "  ENSEMBL lowerbound ENSEMBL upperbound         ENTREZ ENTREZ lowerbound  \\\n",
       "0         45333471.0         45341284.0    ENTREZ:7178        45333471.0   \n",
       "1         48892765.0         48895871.0     ENTREZ:516        48892765.0   \n",
       "2          7040849.0          7043001.0  ENTREZ:257236         7040849.0   \n",
       "3         43492032.0         43492313.0            NaN               NaN   \n",
       "4        107571161.0        107965180.0   ENTREZ:10451       107571161.0   \n",
       "\n",
       "  ENTREZ upperbound PUBCHEM UNIPROTKB  \n",
       "0        45341183.0     NaN    P13693  \n",
       "1        48895871.0     NaN    P05496  \n",
       "2         7043001.0     NaN    Q2M329  \n",
       "3               NaN     NaN       NaN  \n",
       "4       107965180.0     NaN    Q9UKW4  \n",
       "\n",
       "[5 rows x 29 columns]"
      ]
     },
     "execution_count": 111,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hgnc.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/UNIPROTKB.molecularly interacts with.ENCODE RBS HEPG2 K562.edges.csv\n",
      "out/0915/filtered/edges/UNIPROTKB.predicted in.Body Substance.edges.csv\n",
      "out/0915/filtered/edges/UNIPROT.isa.GLYCAN.edges.csv\n",
      "out/0915/filtered/edges/GLYCAN.isa.UNIPROT.edges.csv\n",
      "out/0915/filtered/edges/UNIPROTKB.not predicted in.Body Substance.edges.csv\n",
      "out/0915/filtered/edges/GLYGEN GLYCOSYLATION.has enzyme protein.UNIPROTKB.edges.csv\n",
      "out/0915/filtered/edges/GLYCOSYLTRANSFERASE REACTION.has enzyme protein.UNIPROTKB.edges.csv\n",
      "out/0915/filtered/edges/UNIPROTKB.molecularly interacts with.ENCODE RBS HEPG2.edges.csv\n",
      "out/0915/filtered/edges/Drug.bioactivity.UNIPROTKB.edges.csv\n",
      "out/0915/filtered/edges/UNIPROTKB.molecularly interacts with.ENCODE RBS K562.edges.csv\n"
     ]
    }
   ],
   "source": [
    "for i in glob('out/0915/filtered/edges/*'):\n",
    "    if \"UNIPROT\" in i:\n",
    "        print(i)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'concepts' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[113], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m concepts\u001b[39m.\u001b[39mtype\u001b[39m.\u001b[39munique()\n",
      "\u001b[0;31mNameError\u001b[0m: name 'concepts' is not defined"
     ]
    }
   ],
   "source": [
    "concepts.type.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_30234/1904130140.py:3: DtypeWarning: Columns (15,16,19,27) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  gene_df = pd.read_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\", index_col=0)\n"
     ]
    }
   ],
   "source": [
    "gene_or_genome_df = pd.read_csv(\"out/0915/filtered/nodes/Gene or Genome.nodes.csv\", index_col=0)\n",
    "uniprot = pd.read_csv(\"out/0915/filtered/nodes/UNIPROTKB.nodes.csv\", index_col=0)\n",
    "gene_df = pd.read_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\", index_col=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>MTH</th>\n",
       "      <th>PUBCHEM</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C2828054</th>\n",
       "      <td>FANCD2 wt Allele</td>\n",
       "      <td>Gene or Genome</td>\n",
       "      <td>NCI:C86550</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>PUBCHEM:643975</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                     label            type         NCI         MTH  \\\n",
       "id                                                                   \n",
       "C2828054  FANCD2 wt Allele  Gene or Genome  NCI:C86550  MTH:NOCODE   \n",
       "\n",
       "                 PUBCHEM  \n",
       "id                        \n",
       "C2828054  PUBCHEM:643975  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_or_genome_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P41182 CUI</th>\n",
       "      <td>BCL6_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:P41182</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P51531 CUI</th>\n",
       "      <td>SMCA2_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:P51531</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P25054 CUI</th>\n",
       "      <td>APC_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:P25054</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:Q07817 CUI</th>\n",
       "      <td>B2CL1_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:Q07817</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P19801 CUI</th>\n",
       "      <td>AOC1_HUMAN</td>\n",
       "      <td>UNIPROTKB</td>\n",
       "      <td>UNIPROTKB:P19801</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                            label       type         UNIPROTKB\n",
       "id                                                            \n",
       "UNIPROTKB:P41182 CUI   BCL6_HUMAN  UNIPROTKB  UNIPROTKB:P41182\n",
       "UNIPROTKB:P51531 CUI  SMCA2_HUMAN  UNIPROTKB  UNIPROTKB:P51531\n",
       "UNIPROTKB:P25054 CUI    APC_HUMAN  UNIPROTKB  UNIPROTKB:P25054\n",
       "UNIPROTKB:Q07817 CUI  B2CL1_HUMAN  UNIPROTKB  UNIPROTKB:Q07817\n",
       "UNIPROTKB:P19801 CUI   AOC1_HUMAN  UNIPROTKB  UNIPROTKB:P19801"
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "uniprot.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>LCH_NW</th>\n",
       "      <th>MSH</th>\n",
       "      <th>CSP</th>\n",
       "      <th>SNOMEDCT_US</th>\n",
       "      <th>HGNC</th>\n",
       "      <th>MTH</th>\n",
       "      <th>OMIM</th>\n",
       "      <th>...</th>\n",
       "      <th>CHEBI</th>\n",
       "      <th>ENSEMBL</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "      <th>ENTREZ lowerbound</th>\n",
       "      <th>ENTREZ upperbound</th>\n",
       "      <th>PUBCHEM</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1420882</th>\n",
       "      <td>TPT1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C101389</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12022</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:600763</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000133112</td>\n",
       "      <td>17.0</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341284.0</td>\n",
       "      <td>ENTREZ:7178</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341183.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P13693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1412662</th>\n",
       "      <td>ATP5MC1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:841</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:603192</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000159199</td>\n",
       "      <td>14.0</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>ENTREZ:516</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P05496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1824623</th>\n",
       "      <td>CCDC96 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:26900</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:619347</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000173013</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>ENTREZ:257236</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q2M329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C3543374</th>\n",
       "      <td>RN7SL531P gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:46547</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000264071</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43492032.0</td>\n",
       "      <td>43492313.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1336927</th>\n",
       "      <td>VAV3 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C24894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12659</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:605541</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000134215</td>\n",
       "      <td>16.0</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>ENTREZ:10451</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9UKW4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   label  type          NCI LCH_NW  MSH  CSP SNOMEDCT_US  \\\n",
       "C1420882       TPT1 gene  Gene  NCI:C101389    NaN  NaN  NaN         NaN   \n",
       "C1412662    ATP5MC1 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1824623     CCDC96 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C3543374  RN7SL531P gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1336927       VAV3 gene  Gene   NCI:C24894    NaN  NaN  NaN         NaN   \n",
       "\n",
       "                HGNC         MTH         OMIM  ... CHEBI  \\\n",
       "C1420882  HGNC:12022  MTH:NOCODE  OMIM:600763  ...   NaN   \n",
       "C1412662    HGNC:841  MTH:NOCODE  OMIM:603192  ...   NaN   \n",
       "C1824623  HGNC:26900  MTH:NOCODE  OMIM:619347  ...   NaN   \n",
       "C3543374  HGNC:46547  MTH:NOCODE          NaN  ...   NaN   \n",
       "C1336927  HGNC:12659  MTH:NOCODE  OMIM:605541  ...   NaN   \n",
       "\n",
       "                          ENSEMBL ENSEMBL value ENSEMBL lowerbound  \\\n",
       "C1420882  ENSEMBL:ENSG00000133112          17.0         45333471.0   \n",
       "C1412662  ENSEMBL:ENSG00000159199          14.0         48892765.0   \n",
       "C1824623  ENSEMBL:ENSG00000173013           6.0          7040849.0   \n",
       "C3543374  ENSEMBL:ENSG00000264071           2.0         43492032.0   \n",
       "C1336927  ENSEMBL:ENSG00000134215          16.0        107571161.0   \n",
       "\n",
       "         ENSEMBL upperbound         ENTREZ ENTREZ lowerbound  \\\n",
       "C1420882         45341284.0    ENTREZ:7178        45333471.0   \n",
       "C1412662         48895871.0     ENTREZ:516        48892765.0   \n",
       "C1824623          7043001.0  ENTREZ:257236         7040849.0   \n",
       "C3543374         43492313.0            NaN               NaN   \n",
       "C1336927        107965180.0   ENTREZ:10451       107571161.0   \n",
       "\n",
       "         ENTREZ upperbound PUBCHEM UNIPROTKB  \n",
       "C1420882        45341183.0     NaN    P13693  \n",
       "C1412662        48895871.0     NaN    P05496  \n",
       "C1824623         7043001.0     NaN    Q2M329  \n",
       "C3543374               NaN     NaN       NaN  \n",
       "C1336927       107965180.0     NaN    Q9UKW4  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 117,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in gene_or_genome_df.iterrows():\n",
    "    gene_df.loc[i] = row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in uniprot.iterrows():\n",
    "    gene_df.loc[i] = row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.type = \"Gene\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.to_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(\"out/0915/filtered/nodes/UNIPROTKB.nodes.csv\")\n",
    "os.remove(\"out/0915/filtered/nodes/Gene or Genome.nodes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i in glob('out/0915/filtered/edges/*'):\n",
    "    if \"UNIPROTKB\" in i:\n",
    "        os.rename(i, i.replace(\"UNIPROTKB\", \"Gene\"))\n",
    "    if \"Gene or Genome\" in i:\n",
    "        os.rename(i, i.replace(\"Gene or Genome\", \"Gene\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>LCH_NW</th>\n",
       "      <th>MSH</th>\n",
       "      <th>CSP</th>\n",
       "      <th>SNOMEDCT_US</th>\n",
       "      <th>HGNC</th>\n",
       "      <th>MTH</th>\n",
       "      <th>OMIM</th>\n",
       "      <th>...</th>\n",
       "      <th>CHEBI</th>\n",
       "      <th>ENSEMBL</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "      <th>ENTREZ lowerbound</th>\n",
       "      <th>ENTREZ upperbound</th>\n",
       "      <th>PUBCHEM</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1420882</th>\n",
       "      <td>TPT1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C101389</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12022</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:600763</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000133112</td>\n",
       "      <td>17.0</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341284.0</td>\n",
       "      <td>ENTREZ:7178</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341183.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P13693</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1412662</th>\n",
       "      <td>ATP5MC1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:841</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:603192</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000159199</td>\n",
       "      <td>14.0</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>ENTREZ:516</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P05496</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1824623</th>\n",
       "      <td>CCDC96 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:26900</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:619347</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000173013</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>ENTREZ:257236</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q2M329</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C3543374</th>\n",
       "      <td>RN7SL531P gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:46547</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000264071</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43492032.0</td>\n",
       "      <td>43492313.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1336927</th>\n",
       "      <td>VAV3 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C24894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12659</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:605541</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ENSEMBL:ENSG00000134215</td>\n",
       "      <td>16.0</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>ENTREZ:10451</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9UKW4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 28 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   label  type          NCI LCH_NW  MSH  CSP SNOMEDCT_US  \\\n",
       "C1420882       TPT1 gene  Gene  NCI:C101389    NaN  NaN  NaN         NaN   \n",
       "C1412662    ATP5MC1 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1824623     CCDC96 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C3543374  RN7SL531P gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1336927       VAV3 gene  Gene   NCI:C24894    NaN  NaN  NaN         NaN   \n",
       "\n",
       "                HGNC         MTH         OMIM  ... CHEBI  \\\n",
       "C1420882  HGNC:12022  MTH:NOCODE  OMIM:600763  ...   NaN   \n",
       "C1412662    HGNC:841  MTH:NOCODE  OMIM:603192  ...   NaN   \n",
       "C1824623  HGNC:26900  MTH:NOCODE  OMIM:619347  ...   NaN   \n",
       "C3543374  HGNC:46547  MTH:NOCODE          NaN  ...   NaN   \n",
       "C1336927  HGNC:12659  MTH:NOCODE  OMIM:605541  ...   NaN   \n",
       "\n",
       "                          ENSEMBL ENSEMBL value ENSEMBL lowerbound  \\\n",
       "C1420882  ENSEMBL:ENSG00000133112          17.0         45333471.0   \n",
       "C1412662  ENSEMBL:ENSG00000159199          14.0         48892765.0   \n",
       "C1824623  ENSEMBL:ENSG00000173013           6.0          7040849.0   \n",
       "C3543374  ENSEMBL:ENSG00000264071           2.0         43492032.0   \n",
       "C1336927  ENSEMBL:ENSG00000134215          16.0        107571161.0   \n",
       "\n",
       "         ENSEMBL upperbound         ENTREZ ENTREZ lowerbound  \\\n",
       "C1420882         45341284.0    ENTREZ:7178        45333471.0   \n",
       "C1412662         48895871.0     ENTREZ:516        48892765.0   \n",
       "C1824623          7043001.0  ENTREZ:257236         7040849.0   \n",
       "C3543374         43492313.0            NaN               NaN   \n",
       "C1336927        107965180.0   ENTREZ:10451       107571161.0   \n",
       "\n",
       "         ENTREZ upperbound PUBCHEM UNIPROTKB  \n",
       "C1420882        45341183.0     NaN    P13693  \n",
       "C1412662        48895871.0     NaN    P05496  \n",
       "C1824623         7043001.0     NaN    Q2M329  \n",
       "C3543374               NaN     NaN       NaN  \n",
       "C1336927       107965180.0     NaN    Q9UKW4  \n",
       "\n",
       "[5 rows x 28 columns]"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>HGNC ID</th>\n",
       "      <th>Status</th>\n",
       "      <th>Approved symbol</th>\n",
       "      <th>Approved name</th>\n",
       "      <th>Enzyme (EC) ID</th>\n",
       "      <th>UniProt accession</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>HGNC:5</td>\n",
       "      <td>Approved</td>\n",
       "      <td>A1BG</td>\n",
       "      <td>alpha-1-B glycoprotein</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P04217</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>HGNC:37133</td>\n",
       "      <td>Approved</td>\n",
       "      <td>A1BG-AS1</td>\n",
       "      <td>A1BG antisense RNA 1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>HGNC:24086</td>\n",
       "      <td>Approved</td>\n",
       "      <td>A1CF</td>\n",
       "      <td>APOBEC1 complementation factor</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9NQ94</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>HGNC:7</td>\n",
       "      <td>Approved</td>\n",
       "      <td>A2M</td>\n",
       "      <td>alpha-2-macroglobulin</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P01023</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>HGNC:27057</td>\n",
       "      <td>Approved</td>\n",
       "      <td>A2M-AS1</td>\n",
       "      <td>A2M antisense RNA 1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      HGNC ID    Status Approved symbol                   Approved name  \\\n",
       "0      HGNC:5  Approved            A1BG          alpha-1-B glycoprotein   \n",
       "1  HGNC:37133  Approved        A1BG-AS1            A1BG antisense RNA 1   \n",
       "2  HGNC:24086  Approved            A1CF  APOBEC1 complementation factor   \n",
       "3      HGNC:7  Approved             A2M           alpha-2-macroglobulin   \n",
       "4  HGNC:27057  Approved         A2M-AS1             A2M antisense RNA 1   \n",
       "\n",
       "  Enzyme (EC) ID UniProt accession  \n",
       "0            NaN            P04217  \n",
       "1            NaN               NaN  \n",
       "2            NaN            Q9NQ94  \n",
       "3            NaN            P01023  \n",
       "4            NaN               NaN  "
      ]
     },
     "execution_count": 126,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hgnc_genes = pd.read_csv(\"dd_data/HGNC_genes.txt\", sep=\"\\t\")\n",
    "hgnc_genes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {},
   "outputs": [],
   "source": [
    "hgnc_mapper = {}\n",
    "for i, row in hgnc_genes.iterrows():\n",
    "    hgnc_id = row[\"HGNC ID\"]\n",
    "    enz_id = row[\"Enzyme (EC) ID\"]\n",
    "    if hgnc_id not in hgnc_mapper:\n",
    "        hgnc_mapper[hgnc_id] = {\n",
    "            \"EC ID\": enz_id,\n",
    "            \"is_Enzyme\": type(enz_id) == str\n",
    "        }\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, row in gene_df.iterrows():\n",
    "    hgnc_id = row[\"HGNC\"]\n",
    "    if hgnc_id in hgnc_mapper:\n",
    "        gene_df.at[i, \"EC ID\"] = hgnc_mapper[hgnc_id][\"EC ID\"]\n",
    "        gene_df.at[i, \"is_Enzyme\"] = hgnc_mapper[hgnc_id][\"is_Enzyme\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>LCH_NW</th>\n",
       "      <th>MSH</th>\n",
       "      <th>CSP</th>\n",
       "      <th>SNOMEDCT_US</th>\n",
       "      <th>HGNC</th>\n",
       "      <th>MTH</th>\n",
       "      <th>OMIM</th>\n",
       "      <th>...</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "      <th>ENTREZ lowerbound</th>\n",
       "      <th>ENTREZ upperbound</th>\n",
       "      <th>PUBCHEM</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "      <th>EC ID</th>\n",
       "      <th>is_Enzyme</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1826654</th>\n",
       "      <td>PPCS gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:25686</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:609853</td>\n",
       "      <td>...</td>\n",
       "      <td>9.0</td>\n",
       "      <td>42456117.0</td>\n",
       "      <td>42473385.0</td>\n",
       "      <td>ENTREZ:79717</td>\n",
       "      <td>42456117.0</td>\n",
       "      <td>42460081.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9HAB8</td>\n",
       "      <td>6.3.2.5</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1414049</th>\n",
       "      <td>SEPTIN1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:2879</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:612897</td>\n",
       "      <td>...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>30378135.0</td>\n",
       "      <td>30395991.0</td>\n",
       "      <td>ENTREZ:1731</td>\n",
       "      <td>30378135.0</td>\n",
       "      <td>30382561.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q8WYJ6</td>\n",
       "      <td>3.1.5.1</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1425932</th>\n",
       "      <td>DCXR gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:18985</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:608347</td>\n",
       "      <td>...</td>\n",
       "      <td>8.0</td>\n",
       "      <td>82035136.0</td>\n",
       "      <td>82037709.0</td>\n",
       "      <td>ENTREZ:51181</td>\n",
       "      <td>82035136.0</td>\n",
       "      <td>82036995.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q7Z4W1</td>\n",
       "      <td>1.1.1.10</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1842869</th>\n",
       "      <td>INPP5K gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:33882</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:607875</td>\n",
       "      <td>...</td>\n",
       "      <td>20.0</td>\n",
       "      <td>1494577.0</td>\n",
       "      <td>1516742.0</td>\n",
       "      <td>ENTREZ:51763</td>\n",
       "      <td>1494577.0</td>\n",
       "      <td>1516612.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9BT40</td>\n",
       "      <td>3.1.3.56</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1415276</th>\n",
       "      <td>GPX2 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C104434</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:4554</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:138319</td>\n",
       "      <td>...</td>\n",
       "      <td>13.0</td>\n",
       "      <td>64939152.0</td>\n",
       "      <td>64942746.0</td>\n",
       "      <td>ENTREZ:2877</td>\n",
       "      <td>64939152.0</td>\n",
       "      <td>64942745.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P18283</td>\n",
       "      <td>1.11.1.9</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1825958</th>\n",
       "      <td>MECR gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C89008</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:19691</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:608205</td>\n",
       "      <td>...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>29192657.0</td>\n",
       "      <td>29230942.0</td>\n",
       "      <td>ENTREZ:51102</td>\n",
       "      <td>29192657.0</td>\n",
       "      <td>29230934.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9BV79</td>\n",
       "      <td>1.3.1.38</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1420620</th>\n",
       "      <td>TBXAS1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:11609</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:274180</td>\n",
       "      <td>...</td>\n",
       "      <td>18.0</td>\n",
       "      <td>139777051.0</td>\n",
       "      <td>140020325.0</td>\n",
       "      <td>ENTREZ:6916</td>\n",
       "      <td>139777051.0</td>\n",
       "      <td>139789622.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P24557</td>\n",
       "      <td>5.3.99.5</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1412639</th>\n",
       "      <td>ATP2A3 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:813</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:601929</td>\n",
       "      <td>...</td>\n",
       "      <td>18.0</td>\n",
       "      <td>3923870.0</td>\n",
       "      <td>3964464.0</td>\n",
       "      <td>ENTREZ:489</td>\n",
       "      <td>3923870.0</td>\n",
       "      <td>3964464.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q93084</td>\n",
       "      <td>3.6.3.8</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1537381</th>\n",
       "      <td>HSD17B6 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C105152</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:23316</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:606623</td>\n",
       "      <td>...</td>\n",
       "      <td>11.0</td>\n",
       "      <td>56752161.0</td>\n",
       "      <td>56787790.0</td>\n",
       "      <td>ENTREZ:8630</td>\n",
       "      <td>56752161.0</td>\n",
       "      <td>56787736.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>O14756</td>\n",
       "      <td>1.1.1.62</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1418534</th>\n",
       "      <td>PHKB gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:8927</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:172490</td>\n",
       "      <td>...</td>\n",
       "      <td>18.0</td>\n",
       "      <td>47461123.0</td>\n",
       "      <td>47701523.0</td>\n",
       "      <td>ENTREZ:5257</td>\n",
       "      <td>47461123.0</td>\n",
       "      <td>47547547.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q93100</td>\n",
       "      <td>2.7.11.19</td>\n",
       "      <td>True</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>1890 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                 label  type          NCI LCH_NW  MSH  CSP SNOMEDCT_US  \\\n",
       "C1826654     PPCS gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1414049  SEPTIN1 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1425932     DCXR gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1842869   INPP5K gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1415276     GPX2 gene  Gene  NCI:C104434    NaN  NaN  NaN         NaN   \n",
       "...                ...   ...          ...    ...  ...  ...         ...   \n",
       "C1825958     MECR gene  Gene   NCI:C89008    NaN  NaN  NaN         NaN   \n",
       "C1420620   TBXAS1 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1412639   ATP2A3 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1537381  HSD17B6 gene  Gene  NCI:C105152    NaN  NaN  NaN         NaN   \n",
       "C1418534     PHKB gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "\n",
       "                HGNC         MTH         OMIM  ... ENSEMBL value  \\\n",
       "C1826654  HGNC:25686  MTH:NOCODE  OMIM:609853  ...           9.0   \n",
       "C1414049   HGNC:2879  MTH:NOCODE  OMIM:612897  ...          13.0   \n",
       "C1425932  HGNC:18985  MTH:NOCODE  OMIM:608347  ...           8.0   \n",
       "C1842869  HGNC:33882  MTH:NOCODE  OMIM:607875  ...          20.0   \n",
       "C1415276   HGNC:4554  MTH:NOCODE  OMIM:138319  ...          13.0   \n",
       "...              ...         ...          ...  ...           ...   \n",
       "C1825958  HGNC:19691  MTH:NOCODE  OMIM:608205  ...          16.0   \n",
       "C1420620  HGNC:11609  MTH:NOCODE  OMIM:274180  ...          18.0   \n",
       "C1412639    HGNC:813  MTH:NOCODE  OMIM:601929  ...          18.0   \n",
       "C1537381  HGNC:23316  MTH:NOCODE  OMIM:606623  ...          11.0   \n",
       "C1418534   HGNC:8927  MTH:NOCODE  OMIM:172490  ...          18.0   \n",
       "\n",
       "         ENSEMBL lowerbound ENSEMBL upperbound        ENTREZ  \\\n",
       "C1826654         42456117.0         42473385.0  ENTREZ:79717   \n",
       "C1414049         30378135.0         30395991.0   ENTREZ:1731   \n",
       "C1425932         82035136.0         82037709.0  ENTREZ:51181   \n",
       "C1842869          1494577.0          1516742.0  ENTREZ:51763   \n",
       "C1415276         64939152.0         64942746.0   ENTREZ:2877   \n",
       "...                     ...                ...           ...   \n",
       "C1825958         29192657.0         29230942.0  ENTREZ:51102   \n",
       "C1420620        139777051.0        140020325.0   ENTREZ:6916   \n",
       "C1412639          3923870.0          3964464.0    ENTREZ:489   \n",
       "C1537381         56752161.0         56787790.0   ENTREZ:8630   \n",
       "C1418534         47461123.0         47701523.0   ENTREZ:5257   \n",
       "\n",
       "         ENTREZ lowerbound ENTREZ upperbound PUBCHEM UNIPROTKB      EC ID  \\\n",
       "C1826654        42456117.0        42460081.0     NaN    Q9HAB8    6.3.2.5   \n",
       "C1414049        30378135.0        30382561.0     NaN    Q8WYJ6    3.1.5.1   \n",
       "C1425932        82035136.0        82036995.0     NaN    Q7Z4W1   1.1.1.10   \n",
       "C1842869         1494577.0         1516612.0     NaN    Q9BT40   3.1.3.56   \n",
       "C1415276        64939152.0        64942745.0     NaN    P18283   1.11.1.9   \n",
       "...                    ...               ...     ...       ...        ...   \n",
       "C1825958        29192657.0        29230934.0     NaN    Q9BV79   1.3.1.38   \n",
       "C1420620       139777051.0       139789622.0     NaN    P24557   5.3.99.5   \n",
       "C1412639         3923870.0         3964464.0     NaN    Q93084    3.6.3.8   \n",
       "C1537381        56752161.0        56787736.0     NaN    O14756   1.1.1.62   \n",
       "C1418534        47461123.0        47547547.0     NaN    Q93100  2.7.11.19   \n",
       "\n",
       "         is_Enzyme  \n",
       "C1826654      True  \n",
       "C1414049      True  \n",
       "C1425932      True  \n",
       "C1842869      True  \n",
       "C1415276      True  \n",
       "...            ...  \n",
       "C1825958      True  \n",
       "C1420620      True  \n",
       "C1412639      True  \n",
       "C1537381      True  \n",
       "C1418534      True  \n",
       "\n",
       "[1890 rows x 30 columns]"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_df[gene_df.is_Enzyme == True]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 130,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Gene.negatively correlated with chemical or drug.Hormone.edges.csv\n",
      "out/0915/filtered/edges/Gene.negatively correlated with chemical or drug.Amino Acid, Peptide, or Protein.edges.csv\n",
      "out/0915/filtered/edges/Gene.negatively correlated with chemical or drug.Organic Chemical.edges.csv\n",
      "out/0915/filtered/edges/Gene.negatively correlated with chemical or drug.Drug.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively correlated with chemical or drug.Drug.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively correlated with chemical or drug.Organic Chemical.edges.csv\n",
      "out/0915/filtered/edges/Gene.negatively correlated with chemical or drug.Pharmacologic Substance.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively correlated with chemical or drug.Nucleic Acid, Nucleoside, or Nucleotide.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively correlated with chemical or drug.Pharmacologic Substance.edges.csv\n",
      "out/0915/filtered/edges/Gene.negatively correlated with chemical or drug.Nucleic Acid, Nucleoside, or Nucleotide.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively correlated with chemical or drug.Hormone.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively correlated with chemical or drug.Amino Acid, Peptide, or Protein.edges.csv\n"
     ]
    }
   ],
   "source": [
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    df = pd.read_csv(filename)\n",
    "    if \"CMAP\" in df.SAB.unique():\n",
    "        print(filename)\n",
    "        os.remove(filename)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 131,
   "metadata": {},
   "outputs": [],
   "source": [
    "dcc_mapper = {}\n",
    "with open('output/sabs_dcc_mapper.txt') as o:\n",
    "    for line in o:\n",
    "        r = line.strip().split(\":\")\n",
    "        if len(r) == 2:\n",
    "            dcc_mapper[r[0]] =r[1]\n",
    "        else:\n",
    "            dcc_mapper[r[0]] =r[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 132,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/GLYCAN.isa.UNIPROT.edges.csv ['GLYCORDF' 'GLYCOCOO']\n",
      "out/0915/filtered/edges/GLYCAN.isa.GLYCAN.edges.csv ['GLYCORDF' 'GLYCOCOO']\n"
     ]
    }
   ],
   "source": [
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    df = pd.read_csv(filename)\n",
    "    if len(df.SAB.unique()) > 1:\n",
    "        print(filename, df.SAB.unique())\n",
    "        df[\"DCC\"] = dcc_mapper[df.SAB.unique()[0]]\n",
    "    else:\n",
    "        df[\"DCC\"] = dcc_mapper[df.SAB.unique()[0]]\n",
    "    df.to_csv(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>LCH_NW</th>\n",
       "      <th>MSH</th>\n",
       "      <th>CSP</th>\n",
       "      <th>SNOMEDCT_US</th>\n",
       "      <th>HGNC</th>\n",
       "      <th>MTH</th>\n",
       "      <th>OMIM</th>\n",
       "      <th>...</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "      <th>ENTREZ lowerbound</th>\n",
       "      <th>ENTREZ upperbound</th>\n",
       "      <th>PUBCHEM</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "      <th>EC ID</th>\n",
       "      <th>is_Enzyme</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1420882</th>\n",
       "      <td>TPT1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C101389</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12022</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:600763</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341284.0</td>\n",
       "      <td>ENTREZ:7178</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341183.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P13693</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1412662</th>\n",
       "      <td>ATP5MC1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:841</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:603192</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>ENTREZ:516</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P05496</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1824623</th>\n",
       "      <td>CCDC96 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:26900</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:619347</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>ENTREZ:257236</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q2M329</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C3543374</th>\n",
       "      <td>RN7SL531P gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:46547</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43492032.0</td>\n",
       "      <td>43492313.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1336927</th>\n",
       "      <td>VAV3 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C24894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12659</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:605541</td>\n",
       "      <td>...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>ENTREZ:10451</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9UKW4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P16083 CUI</th>\n",
       "      <td>NQO2_HUMAN</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNIPROTKB:P16083</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P10646 CUI</th>\n",
       "      <td>TFPI1_HUMAN</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNIPROTKB:P10646</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:Q92772 CUI</th>\n",
       "      <td>CDKL2_HUMAN</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNIPROTKB:Q92772</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P01375 CUI</th>\n",
       "      <td>TNFA_HUMAN</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNIPROTKB:P01375</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>UNIPROTKB:P17655 CUI</th>\n",
       "      <td>CAN2_HUMAN</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>UNIPROTKB:P17655</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>42467 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                               label  type          NCI LCH_NW  MSH  CSP  \\\n",
       "id                                                                         \n",
       "C1420882                   TPT1 gene  Gene  NCI:C101389    NaN  NaN  NaN   \n",
       "C1412662                ATP5MC1 gene  Gene          NaN    NaN  NaN  NaN   \n",
       "C1824623                 CCDC96 gene  Gene          NaN    NaN  NaN  NaN   \n",
       "C3543374              RN7SL531P gene  Gene          NaN    NaN  NaN  NaN   \n",
       "C1336927                   VAV3 gene  Gene   NCI:C24894    NaN  NaN  NaN   \n",
       "...                              ...   ...          ...    ...  ...  ...   \n",
       "UNIPROTKB:P16083 CUI      NQO2_HUMAN  Gene          NaN    NaN  NaN  NaN   \n",
       "UNIPROTKB:P10646 CUI     TFPI1_HUMAN  Gene          NaN    NaN  NaN  NaN   \n",
       "UNIPROTKB:Q92772 CUI     CDKL2_HUMAN  Gene          NaN    NaN  NaN  NaN   \n",
       "UNIPROTKB:P01375 CUI      TNFA_HUMAN  Gene          NaN    NaN  NaN  NaN   \n",
       "UNIPROTKB:P17655 CUI      CAN2_HUMAN  Gene          NaN    NaN  NaN  NaN   \n",
       "\n",
       "                     SNOMEDCT_US        HGNC         MTH         OMIM  ...  \\\n",
       "id                                                                     ...   \n",
       "C1420882                     NaN  HGNC:12022  MTH:NOCODE  OMIM:600763  ...   \n",
       "C1412662                     NaN    HGNC:841  MTH:NOCODE  OMIM:603192  ...   \n",
       "C1824623                     NaN  HGNC:26900  MTH:NOCODE  OMIM:619347  ...   \n",
       "C3543374                     NaN  HGNC:46547  MTH:NOCODE          NaN  ...   \n",
       "C1336927                     NaN  HGNC:12659  MTH:NOCODE  OMIM:605541  ...   \n",
       "...                          ...         ...         ...          ...  ...   \n",
       "UNIPROTKB:P16083 CUI         NaN         NaN         NaN          NaN  ...   \n",
       "UNIPROTKB:P10646 CUI         NaN         NaN         NaN          NaN  ...   \n",
       "UNIPROTKB:Q92772 CUI         NaN         NaN         NaN          NaN  ...   \n",
       "UNIPROTKB:P01375 CUI         NaN         NaN         NaN          NaN  ...   \n",
       "UNIPROTKB:P17655 CUI         NaN         NaN         NaN          NaN  ...   \n",
       "\n",
       "                     ENSEMBL value ENSEMBL lowerbound ENSEMBL upperbound  \\\n",
       "id                                                                         \n",
       "C1420882                      17.0         45333471.0         45341284.0   \n",
       "C1412662                      14.0         48892765.0         48895871.0   \n",
       "C1824623                       6.0          7040849.0          7043001.0   \n",
       "C3543374                       2.0         43492032.0         43492313.0   \n",
       "C1336927                      16.0        107571161.0        107965180.0   \n",
       "...                            ...                ...                ...   \n",
       "UNIPROTKB:P16083 CUI           NaN                NaN                NaN   \n",
       "UNIPROTKB:P10646 CUI           NaN                NaN                NaN   \n",
       "UNIPROTKB:Q92772 CUI           NaN                NaN                NaN   \n",
       "UNIPROTKB:P01375 CUI           NaN                NaN                NaN   \n",
       "UNIPROTKB:P17655 CUI           NaN                NaN                NaN   \n",
       "\n",
       "                             ENTREZ ENTREZ lowerbound ENTREZ upperbound  \\\n",
       "id                                                                        \n",
       "C1420882                ENTREZ:7178        45333471.0        45341183.0   \n",
       "C1412662                 ENTREZ:516        48892765.0        48895871.0   \n",
       "C1824623              ENTREZ:257236         7040849.0         7043001.0   \n",
       "C3543374                        NaN               NaN               NaN   \n",
       "C1336927               ENTREZ:10451       107571161.0       107965180.0   \n",
       "...                             ...               ...               ...   \n",
       "UNIPROTKB:P16083 CUI            NaN               NaN               NaN   \n",
       "UNIPROTKB:P10646 CUI            NaN               NaN               NaN   \n",
       "UNIPROTKB:Q92772 CUI            NaN               NaN               NaN   \n",
       "UNIPROTKB:P01375 CUI            NaN               NaN               NaN   \n",
       "UNIPROTKB:P17655 CUI            NaN               NaN               NaN   \n",
       "\n",
       "                     PUBCHEM         UNIPROTKB EC ID is_Enzyme  \n",
       "id                                                              \n",
       "C1420882                 NaN            P13693   NaN     False  \n",
       "C1412662                 NaN            P05496   NaN     False  \n",
       "C1824623                 NaN            Q2M329   NaN     False  \n",
       "C3543374                 NaN               NaN   NaN     False  \n",
       "C1336927                 NaN            Q9UKW4   NaN     False  \n",
       "...                      ...               ...   ...       ...  \n",
       "UNIPROTKB:P16083 CUI     NaN  UNIPROTKB:P16083   NaN       NaN  \n",
       "UNIPROTKB:P10646 CUI     NaN  UNIPROTKB:P10646   NaN       NaN  \n",
       "UNIPROTKB:Q92772 CUI     NaN  UNIPROTKB:Q92772   NaN       NaN  \n",
       "UNIPROTKB:P01375 CUI     NaN  UNIPROTKB:P01375   NaN       NaN  \n",
       "UNIPROTKB:P17655 CUI     NaN  UNIPROTKB:P17655   NaN       NaN  \n",
       "\n",
       "[42467 rows x 30 columns]"
      ]
     },
     "execution_count": 133,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_df.index.name = \"id\"\n",
    "gene_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 134,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.to_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 135,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/nodes/GLYCOSYLTRANSFERASE REACTION.nodes.csv\n",
      "out/0915/filtered/nodes/EXPBINS.nodes.csv\n",
      "out/0915/filtered/nodes/GLYGEN SRC.nodes.csv\n",
      "out/0915/filtered/nodes/GLYGEN GLYCOSYLATION.nodes.csv\n",
      "out/0915/filtered/nodes/CHLO.nodes.csv\n",
      "out/0915/filtered/nodes/ENCODE CCRE CTCF.nodes.csv\n",
      "out/0915/filtered/nodes/4DNQ.nodes.csv\n",
      "out/0915/filtered/nodes/4DNL.nodes.csv\n",
      "out/0915/filtered/nodes/MEDGEN.nodes.csv\n",
      "out/0915/filtered/nodes/KFVARBIN.nodes.csv\n",
      "out/0915/filtered/nodes/GTEXEXP.nodes.csv\n",
      "out/0915/filtered/nodes/PVALUEBINS.nodes.csv\n",
      "out/0915/filtered/nodes/KFPT.nodes.csv\n",
      "out/0915/filtered/nodes/GTEXEQTL.nodes.csv\n",
      "out/0915/filtered/nodes/MOTORPAC.nodes.csv\n",
      "out/0915/filtered/nodes/ENCODE CCRE H3K4ME3.nodes.csv\n",
      "out/0915/filtered/nodes/ENCODE CCRE H3K27AC.nodes.csv\n",
      "out/0915/filtered/nodes/ENCODE CCRE.nodes.csv\n"
     ]
    }
   ],
   "source": [
    "filenames = []\n",
    "for filename in glob('out/0915/filtered/nodes/*'):\n",
    "    df = pd.read_csv(filename, index_col=0, low_memory=False)\n",
    "    if \"label\" not in df.columns:\n",
    "        filenames.append(filename)\n",
    "        df['label'] = df.index\n",
    "        df.to_csv(filename)\n",
    "        print(filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18"
      ]
     },
     "execution_count": 136,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(filenames)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [],
   "source": [
    "from py2neo import Graph\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "graph = Graph(os.getenv('NEO4j_URL'), auth=(os.getenv('NEO4J_USER'), os.getenv('NEO4J_PASSWORD')))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [],
   "source": [
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GLYCOSYLTRANSFERASE REACTION\n",
      "EXPBINS\n",
      "GLYGEN SRC\n",
      "GLYGEN GLYCOSYLATION\n",
      "CHLO\n",
      "ENCODE CCRE CTCF\n",
      "4DNQ\n",
      "4DNL\n",
      "MEDGEN\n",
      "KFVARBIN\n",
      "GTEXEXP\n",
      "PVALUEBINS\n",
      "KFPT\n",
      "GTEXEQTL\n",
      "MOTORPAC\n",
      "ENCODE CCRE H3K4ME3\n",
      "ENCODE CCRE H3K27AC\n",
      "ENCODE CCRE\n"
     ]
    }
   ],
   "source": [
    "node_pattern = \"(?P<directory>.+)/(?P<node_type>.+)\\.(?P<entity>.+)\\.csv\"\n",
    "\n",
    "for filename in filenames:\n",
    "    match = re.match(node_pattern, filename).groupdict()\n",
    "    node_type = match[\"node_type\"]\n",
    "    print(node_type)\n",
    "    query = \"MATCH (a: `%s`) WHERE a.label IS NULL SET a.label = a.id\"%node_type\n",
    "    graph.run(query)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 137,
   "metadata": {},
   "outputs": [],
   "source": [
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    df = pd.read_csv(filename, index_col=0)\n",
    "    df.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 138,
   "metadata": {},
   "outputs": [],
   "source": [
    "relations = set()\n",
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    match = re.match(edge_pattern, filename).groupdict()\n",
    "    relations.add(match['relation'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {},
   "outputs": [],
   "source": [
    "from glob import glob\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 140,
   "metadata": {},
   "outputs": [],
   "source": [
    "gtex = \"GTEXEXP\"\n",
    "# gtex = \"GTEXEQTL\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/GTEXEXP.expressed in.Tissue.edges.csv expressed in\n",
      "out/0915/filtered/edges/Gene.expresses.GTEXEXP.edges.csv expresses\n",
      "out/0915/filtered/edges/GTEXEXP.expressed in.Gene.edges.csv expressed in\n",
      "out/0915/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv has expression\n",
      "out/0915/filtered/edges/GTEXEXP.expressed in.Body Location or Region.edges.csv expressed in\n",
      "out/0915/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv expresses\n",
      "out/0915/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv expresses\n",
      "out/0915/filtered/edges/GTEXEXP.expressed in.Body Part, Organ, or Organ Component.edges.csv expressed in\n",
      "out/0915/filtered/edges/Body Location or Region.expresses.GTEXEXP.edges.csv expresses\n"
     ]
    }
   ],
   "source": [
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    # relations.add(match['relation'])\n",
    "    if gtex in filename:\n",
    "        match = re.match(edge_pattern, filename).groupdict()\n",
    "        print(filename, match[\"relation\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene2gtexp = pd.read_csv(\"out/0915/filtered/edges/Gene.expresses.GTEXEXP.edges.csv\")\n",
    "tissue2gtexp = pd.read_csv(\"out/0915/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv\")\n",
    "organ2gtexp = pd.read_csv(\"out/0915/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv\")\n",
    "location2gtexp = pd.read_csv(\"out/0915/filtered/edges/Body Location or Region.expresses.GTEXEXP.edges.csv\")\n",
    "hasExp = pd.read_csv(\"out/0915/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 143,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C2239334</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation                                             target  \\\n",
       "0  C2239334  expresses  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...   \n",
       "1  C2239334  expresses        GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI   \n",
       "2  C2239334  expresses         GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI   \n",
       "3  C2239334  expresses      GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI   \n",
       "4  C2239334  expresses        GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI   \n",
       "\n",
       "   source_label  target_label      SAB  evidence   DCC  \n",
       "0  DDX11L1 gene           NaN  GTEXEXP       NaN  GTEX  \n",
       "1  DDX11L1 gene           NaN  GTEXEXP       NaN  GTEX  \n",
       "2  DDX11L1 gene           NaN  GTEXEXP       NaN  GTEX  \n",
       "3  DDX11L1 gene           NaN  GTEXEXP       NaN  GTEX  \n",
       "4  DDX11L1 gene           NaN  GTEXEXP       NaN  GTEX  "
      ]
     },
     "execution_count": 143,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene2gtexp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0222331</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...</td>\n",
       "      <td>Subcutaneous Fat</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UBERON:0008367 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...</td>\n",
       "      <td>breast epithelium</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>UBERON:EFO 0002009 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UBERON:EFO 0000572 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C1707950</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI</td>\n",
       "      <td>Esophageal Squamous Epithelium</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   source   relation  \\\n",
       "0                C0222331  expresses   \n",
       "1      UBERON:0008367 CUI  expresses   \n",
       "2  UBERON:EFO 0002009 CUI  expresses   \n",
       "3  UBERON:EFO 0000572 CUI  expresses   \n",
       "4                C1707950  expresses   \n",
       "\n",
       "                                              target  \\\n",
       "0  GTEXEXP:ENSG00000223972-5-Adipose-Subcutaneous...   \n",
       "1  GTEXEXP:ENSG00000223972-5-Breast-Mammary-Tissu...   \n",
       "2  GTEXEXP:ENSG00000223972-5-Cells-Cultured-fibro...   \n",
       "3  GTEXEXP:ENSG00000223972-5-Cells-EBV-transforme...   \n",
       "4     GTEXEXP:ENSG00000223972-5-Esophagus-Mucosa CUI   \n",
       "\n",
       "                     source_label  target_label      SAB  evidence   DCC  \n",
       "0                Subcutaneous Fat           NaN  GTEXEXP       NaN  GTEX  \n",
       "1               breast epithelium           NaN  GTEXEXP       NaN  GTEX  \n",
       "2                             NaN           NaN  GTEXEXP       NaN  GTEX  \n",
       "3                             NaN           NaN  GTEXEXP       NaN  GTEX  \n",
       "4  Esophageal Squamous Epithelium           NaN  GTEXEXP       NaN  GTEX  "
      ]
     },
     "execution_count": 144,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tissue2gtexp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0001625</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI</td>\n",
       "      <td>Adrenal Glands</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0003956</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI</td>\n",
       "      <td>Ascending aorta structure</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0205042</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI</td>\n",
       "      <td>Coronary artery</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0085427</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI</td>\n",
       "      <td>Tibial Arteries</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0005682</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Bladder CUI</td>\n",
       "      <td>Urinary Bladder</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation                                         target  \\\n",
       "0  C0001625  expresses    GTEXEXP:ENSG00000223972-5-Adrenal-Gland CUI   \n",
       "1  C0003956  expresses     GTEXEXP:ENSG00000223972-5-Artery-Aorta CUI   \n",
       "2  C0205042  expresses  GTEXEXP:ENSG00000223972-5-Artery-Coronary CUI   \n",
       "3  C0085427  expresses    GTEXEXP:ENSG00000223972-5-Artery-Tibial CUI   \n",
       "4  C0005682  expresses          GTEXEXP:ENSG00000223972-5-Bladder CUI   \n",
       "\n",
       "                source_label  target_label      SAB  evidence   DCC  \n",
       "0             Adrenal Glands           NaN  GTEXEXP       NaN  GTEX  \n",
       "1  Ascending aorta structure           NaN  GTEXEXP       NaN  GTEX  \n",
       "2            Coronary artery           NaN  GTEXEXP       NaN  GTEX  \n",
       "3            Tibial Arteries           NaN  GTEXEXP       NaN  GTEX  \n",
       "4            Urinary Bladder           NaN  GTEXEXP       NaN  GTEX  "
      ]
     },
     "execution_count": 145,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "organ2gtexp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000278267-1-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000243485-5-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>GTEXEXP:ENSG00000237613-2-Kidney-Medulla CUI</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation                                        target  \\\n",
       "0  C0736435  expresses  GTEXEXP:ENSG00000223972-5-Kidney-Medulla CUI   \n",
       "1  C0736435  expresses  GTEXEXP:ENSG00000227232-5-Kidney-Medulla CUI   \n",
       "2  C0736435  expresses  GTEXEXP:ENSG00000278267-1-Kidney-Medulla CUI   \n",
       "3  C0736435  expresses  GTEXEXP:ENSG00000243485-5-Kidney-Medulla CUI   \n",
       "4  C0736435  expresses  GTEXEXP:ENSG00000237613-2-Kidney-Medulla CUI   \n",
       "\n",
       "                            source_label  target_label      SAB  evidence  \\\n",
       "0  Set of outer region of renal pyramids           NaN  GTEXEXP       NaN   \n",
       "1  Set of outer region of renal pyramids           NaN  GTEXEXP       NaN   \n",
       "2  Set of outer region of renal pyramids           NaN  GTEXEXP       NaN   \n",
       "3  Set of outer region of renal pyramids           NaN  GTEXEXP       NaN   \n",
       "4  Set of outer region of renal pyramids           NaN  GTEXEXP       NaN   \n",
       "\n",
       "    DCC  \n",
       "0  GTEX  \n",
       "1  GTEX  \n",
       "2  GTEX  \n",
       "3  GTEX  \n",
       "4  GTEX  "
      ]
     },
     "execution_count": 146,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "location2gtexp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1573785, 8)"
      ]
     },
     "execution_count": 147,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene2gtexp.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEXP:ENSG00000223972-5-Testis CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:0.1.0.2 CUI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:2.0.3.0 CUI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:4.0.5.0 CUI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI</td>\n",
       "      <td>has_expression</td>\n",
       "      <td>EXPBINS:3.0.4.0 CUI</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source        relation  \\\n",
       "0               GTEXEXP:ENSG00000223972-5-Testis CUI  has_expression   \n",
       "1  GTEXEXP:ENSG00000227232-5-Adipose-Subcutaneous...  has_expression   \n",
       "2        GTEXEXP:ENSG00000227232-5-Adrenal-Gland CUI  has_expression   \n",
       "3         GTEXEXP:ENSG00000227232-5-Artery-Aorta CUI  has_expression   \n",
       "4      GTEXEXP:ENSG00000227232-5-Artery-Coronary CUI  has_expression   \n",
       "\n",
       "                target  source_label  target_label      SAB  evidence   DCC  \n",
       "0  EXPBINS:0.1.0.2 CUI           NaN           NaN  GTEXEXP       NaN  GTEX  \n",
       "1  EXPBINS:4.0.5.0 CUI           NaN           NaN  GTEXEXP       NaN  GTEX  \n",
       "2  EXPBINS:2.0.3.0 CUI           NaN           NaN  GTEXEXP       NaN  GTEX  \n",
       "3  EXPBINS:4.0.5.0 CUI           NaN           NaN  GTEXEXP       NaN  GTEX  \n",
       "4  EXPBINS:3.0.4.0 CUI           NaN           NaN  GTEXEXP       NaN  GTEX  "
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hasExp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(244832, 244811)"
      ]
     },
     "execution_count": 149,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(set(tissue2gtexp.target)), len(set(tissue2gtexp.target).intersection(gene2gtexp.target))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1573784, 1573785)"
      ]
     },
     "execution_count": 150,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gtexp_gene_mapper = {}\n",
    "for i, row in gene2gtexp.iterrows():\n",
    "    gene_id = row[\"source\"]\n",
    "    gene = row[\"source_label\"]\n",
    "    gtexp = row[\"target\"]\n",
    "    gtexp_gene_mapper[gtexp] = {\n",
    "        \"gene_id\": gene_id,\n",
    "        \"gene\": gene,\n",
    "    }\n",
    "(i, len(gtexp_gene_mapper))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 151,
   "metadata": {},
   "outputs": [],
   "source": [
    "evidence_mapper = {}\n",
    "for i, row in hasExp.iterrows():\n",
    "    gtexp = row[\"source\"]\n",
    "    target = row[\"target\"]\n",
    "    # EXPBINS:0.1.0.2 CUI\t\n",
    "    score = float(\".\".join(target.replace(\"CUI\", \"\").strip().split(\".\")[2:]))\n",
    "    evidence_mapper[gtexp] = score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 152,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "21\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0222331</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Subcutaneous Fat</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>UBERON:0008367 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>breast epithelium</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>UBERON:EFO 0002009 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>UBERON:EFO 0000572 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>NaN</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C1707950</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Esophageal Squamous Epithelium</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                   source   relation    target  \\\n",
       "0                C0222331  expresses  C2239334   \n",
       "1      UBERON:0008367 CUI  expresses  C2239334   \n",
       "2  UBERON:EFO 0002009 CUI  expresses  C2239334   \n",
       "3  UBERON:EFO 0000572 CUI  expresses  C2239334   \n",
       "4                C1707950  expresses  C2239334   \n",
       "\n",
       "                     source_label  target_label      SAB  evidence   DCC  \n",
       "0                Subcutaneous Fat  DDX11L1 gene  GTEXEXP       0.0  GTEX  \n",
       "1               breast epithelium  DDX11L1 gene  GTEXEXP       0.0  GTEX  \n",
       "2                             NaN  DDX11L1 gene  GTEXEXP       0.0  GTEX  \n",
       "3                             NaN  DDX11L1 gene  GTEXEXP       0.0  GTEX  \n",
       "4  Esophageal Squamous Epithelium  DDX11L1 gene  GTEXEXP       0.0  GTEX  "
      ]
     },
     "execution_count": 152,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter = 0\n",
    "for i, row in tissue2gtexp.iterrows():\n",
    "    target = row[\"target\"]\n",
    "    if target in gtexp_gene_mapper:\n",
    "        val = gtexp_gene_mapper[target]\n",
    "        tissue2gtexp.at[i, 'target'] = val[\"gene_id\"]\n",
    "        tissue2gtexp.at[i, 'target_label'] = val[\"gene\"]\n",
    "        if target in evidence_mapper:\n",
    "            tissue2gtexp.at[i, 'evidence'] = evidence_mapper[target]\n",
    "    else:\n",
    "        counter+=1\n",
    "print(counter)\n",
    "tissue2gtexp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [],
   "source": [
    "tissue2gtexp.to_csv(\"out/0915/filtered/edges/Tissue.expresses.Gene.edges.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "111\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0001625</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Adrenal Glands</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0003956</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Ascending aorta structure</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0205042</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Coronary artery</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0085427</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Tibial Arteries</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0005682</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Urinary Bladder</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation    target               source_label  target_label  \\\n",
       "0  C0001625  expresses  C2239334             Adrenal Glands  DDX11L1 gene   \n",
       "1  C0003956  expresses  C2239334  Ascending aorta structure  DDX11L1 gene   \n",
       "2  C0205042  expresses  C2239334            Coronary artery  DDX11L1 gene   \n",
       "3  C0085427  expresses  C2239334            Tibial Arteries  DDX11L1 gene   \n",
       "4  C0005682  expresses  C2239334            Urinary Bladder  DDX11L1 gene   \n",
       "\n",
       "       SAB  evidence   DCC  \n",
       "0  GTEXEXP       0.0  GTEX  \n",
       "1  GTEXEXP       0.0  GTEX  \n",
       "2  GTEXEXP       0.0  GTEX  \n",
       "3  GTEXEXP       0.0  GTEX  \n",
       "4  GTEXEXP       0.0  GTEX  "
      ]
     },
     "execution_count": 154,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter = 0\n",
    "for i, row in organ2gtexp.iterrows():\n",
    "    target = row[\"target\"]\n",
    "    if target in gtexp_gene_mapper:\n",
    "        val = gtexp_gene_mapper[target]\n",
    "        organ2gtexp.at[i, 'target'] = val[\"gene_id\"]\n",
    "        organ2gtexp.at[i, 'target_label'] = val[\"gene\"]\n",
    "        if target in evidence_mapper:\n",
    "            organ2gtexp.at[i, 'evidence'] = evidence_mapper[target]\n",
    "    else:\n",
    "        counter+=1\n",
    "print(counter)\n",
    "organ2gtexp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [],
   "source": [
    "organ2gtexp.to_csv(\"out/0915/filtered/edges/Body Part, Organ, or Organ Component.expresses.Gene.edges.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 156,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2829144</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>WASH7P gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>3.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C3815338</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>MIR6859-1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C4320459</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>MIR1302-2HG gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0736435</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239429</td>\n",
       "      <td>Set of outer region of renal pyramids</td>\n",
       "      <td>FAM138A gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation    target                           source_label  \\\n",
       "0  C0736435  expresses  C2239334  Set of outer region of renal pyramids   \n",
       "1  C0736435  expresses  C2829144  Set of outer region of renal pyramids   \n",
       "2  C0736435  expresses  C3815338  Set of outer region of renal pyramids   \n",
       "3  C0736435  expresses  C4320459  Set of outer region of renal pyramids   \n",
       "4  C0736435  expresses  C2239429  Set of outer region of renal pyramids   \n",
       "\n",
       "       target_label      SAB  evidence   DCC  \n",
       "0      DDX11L1 gene  GTEXEXP       0.0  GTEX  \n",
       "1       WASH7P gene  GTEXEXP       3.0  GTEX  \n",
       "2    MIR6859-1 gene  GTEXEXP       0.0  GTEX  \n",
       "3  MIR1302-2HG gene  GTEXEXP       0.0  GTEX  \n",
       "4      FAM138A gene  GTEXEXP       0.0  GTEX  "
      ]
     },
     "execution_count": 156,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "counter = 0\n",
    "for i, row in location2gtexp.iterrows():\n",
    "    target = row[\"target\"]\n",
    "    if target in gtexp_gene_mapper:\n",
    "        val = gtexp_gene_mapper[target]\n",
    "        location2gtexp.at[i, 'target'] = val[\"gene_id\"]\n",
    "        location2gtexp.at[i, 'target_label'] = val[\"gene\"]\n",
    "        if target in evidence_mapper:\n",
    "            location2gtexp.at[i, 'evidence'] = evidence_mapper[target]\n",
    "    else:\n",
    "        counter+=1\n",
    "print(counter)\n",
    "location2gtexp.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [],
   "source": [
    "location2gtexp.to_csv(\"out/0915/filtered/edges/Body Location or Region.expresses.Gene.edges.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 158,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove(\"out/0915/filtered/edges/Gene.expresses.GTEXEXP.edges.csv\")\n",
    "os.remove(\"out/0915/filtered/edges/Tissue.expresses.GTEXEXP.edges.csv\")\n",
    "os.remove(\"out/0915/filtered/edges/Body Part, Organ, or Organ Component.expresses.GTEXEXP.edges.csv\")\n",
    "os.remove(\"out/0915/filtered/edges/Body Location or Region.expresses.GTEXEXP.edges.csv\")\n",
    "os.remove(\"out/0915/filtered/edges/GTEXEXP.has expression.EXPBINS.edges.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 159,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Gene.positively regulated by.GTEXEQTL.edges.csv positively regulated by\n",
      "out/0915/filtered/edges/ENTREZ.positively regulated by.GTEXEQTL.edges.csv positively regulated by\n",
      "out/0915/filtered/edges/Tissue.part of.GTEXEQTL.edges.csv part of\n",
      "out/0915/filtered/edges/ENSEMBL.positively regulated by.GTEXEQTL.edges.csv positively regulated by\n",
      "out/0915/filtered/edges/Tissue.location of.GTEXEQTL.edges.csv location of\n",
      "out/0915/filtered/edges/Gene.location of.GTEXEQTL.edges.csv location of\n",
      "out/0915/filtered/edges/Body Part, Organ, or Organ Component.location of.GTEXEQTL.edges.csv location of\n",
      "out/0915/filtered/edges/ENSEMBL.negatively regulated by.GTEXEQTL.edges.csv negatively regulated by\n",
      "out/0915/filtered/edges/Body Part, Organ, or Organ Component.part of.GTEXEQTL.edges.csv part of\n",
      "out/0915/filtered/edges/EFO.location of.GTEXEQTL.edges.csv location of\n",
      "out/0915/filtered/edges/CHLO.location of.GTEXEQTL.edges.csv location of\n",
      "out/0915/filtered/edges/CLINGEN ALLELE REGISTRY.part of.GTEXEQTL.edges.csv part of\n",
      "out/0915/filtered/edges/Gene.negatively regulated by.GTEXEQTL.edges.csv negatively regulated by\n",
      "out/0915/filtered/edges/Cell.location of.GTEXEQTL.edges.csv location of\n"
     ]
    }
   ],
   "source": [
    "gtex = \"GTEXEQTL\"\n",
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    # relations.add(match['relation'])\n",
    "    match = re.match(edge_pattern, filename).groupdict()\n",
    "    if gtex in match[\"target_type\"]:\n",
    "        print(filename, match[\"relation\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/GTEXEQTL.has part.Tissue.edges.csv has part\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Body Part, Organ, or Organ Component.edges.csv located in\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.ENSEMBL.edges.csv positively regulates\n",
      "out/0915/filtered/edges/GTEXEQTL.has part.Body Part, Organ, or Organ Component.edges.csv has part\n",
      "out/0915/filtered/edges/GTEXEQTL.negatively regulates.Gene.edges.csv negatively regulates\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Cell.edges.csv located in\n",
      "out/0915/filtered/edges/GTEXEQTL.p value.PVALUEBINS.edges.csv p value\n",
      "out/0915/filtered/edges/GTEXEQTL.has part.CLINGEN ALLELE REGISTRY.edges.csv has part\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Gene.edges.csv located in\n",
      "out/0915/filtered/edges/GTEXEQTL.negatively regulates.ENSEMBL.edges.csv negatively regulates\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.ENTREZ.edges.csv positively regulates\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.Gene.edges.csv positively regulates\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.CHLO.edges.csv located in\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Tissue.edges.csv located in\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.EFO.edges.csv located in\n"
     ]
    }
   ],
   "source": [
    "gtex = \"GTEXEQTL\"\n",
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    # relations.add(match['relation'])\n",
    "    match = re.match(edge_pattern, filename).groupdict()\n",
    "    if gtex in match[\"source_type\"]:\n",
    "        print(filename, match[\"relation\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 161,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>ENSEMBL:ENSG00000274253 CUI</td>\n",
       "      <td>positively_regulated_by</td>\n",
       "      <td>GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Pancreas CUI</td>\n",
       "      <td>ENSG00000274253</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCCREG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>ENSEMBL:ENSG00000274253 CUI</td>\n",
       "      <td>positively_regulated_by</td>\n",
       "      <td>GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Cau...</td>\n",
       "      <td>ENSG00000274253</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCCREG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>ENSEMBL:ENSG00000274253 CUI</td>\n",
       "      <td>positively_regulated_by</td>\n",
       "      <td>GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Put...</td>\n",
       "      <td>ENSG00000274253</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCCREG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>ENSEMBL:ENSG00000274253 CUI</td>\n",
       "      <td>positively_regulated_by</td>\n",
       "      <td>GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Hip...</td>\n",
       "      <td>ENSG00000274253</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCCREG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCC</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>ENSEMBL:ENSG00000274253 CUI</td>\n",
       "      <td>positively_regulated_by</td>\n",
       "      <td>GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Cer...</td>\n",
       "      <td>ENSG00000274253</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCCREG</td>\n",
       "      <td>NaN</td>\n",
       "      <td>ERCC</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                        source                 relation  \\\n",
       "0  ENSEMBL:ENSG00000274253 CUI  positively_regulated_by   \n",
       "1  ENSEMBL:ENSG00000274253 CUI  positively_regulated_by   \n",
       "2  ENSEMBL:ENSG00000274253 CUI  positively_regulated_by   \n",
       "3  ENSEMBL:ENSG00000274253 CUI  positively_regulated_by   \n",
       "4  ENSEMBL:ENSG00000274253 CUI  positively_regulated_by   \n",
       "\n",
       "                                              target     source_label  \\\n",
       "0  GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Pancreas CUI  ENSG00000274253   \n",
       "1  GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Cau...  ENSG00000274253   \n",
       "2  GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Put...  ENSG00000274253   \n",
       "3  GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Hip...  ENSG00000274253   \n",
       "4  GTEXEQTL:eQTL.chr15.22757689.A.G.b38.Brain.Cer...  ENSG00000274253   \n",
       "\n",
       "   target_label      SAB  evidence   DCC  \n",
       "0           NaN  ERCCREG       NaN  ERCC  \n",
       "1           NaN  ERCCREG       NaN  ERCC  \n",
       "2           NaN  ERCCREG       NaN  ERCC  \n",
       "3           NaN  ERCCREG       NaN  ERCC  \n",
       "4           NaN  ERCCREG       NaN  ERCC  "
      ]
     },
     "execution_count": 161,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"out/0915/filtered/edges/ENTREZ.positively regulated by.GTEXEQTL.edges.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 162,
   "metadata": {},
   "outputs": [],
   "source": [
    "import csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.regulates.ENTREZ.edges.csv ERCCREG\n",
      "(107, 8)\n",
      "out/0915/filtered/edges/ENTREZ.positively regulated by.Drug.edges.csv LINCS\n",
      "(3, 8)\n",
      "out/0915/filtered/edges/ENTREZ.positively regulated by.GTEXEQTL.edges.csv ERCCREG\n",
      "(8, 8)\n",
      "out/0915/filtered/edges/ENTREZ.regulated by.ENCODE CCRE ACTIVITY.edges.csv ERCCREG\n",
      "(107, 8)\n",
      "out/0915/filtered/edges/Drug.positively regulates.ENTREZ.edges.csv LINCS\n",
      "(3, 8)\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.ENTREZ.edges.csv ERCCREG\n",
      "(8, 8)\n"
     ]
    }
   ],
   "source": [
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    if 'ENTREZ.' in filename:\n",
    "        with open(filename) as o:\n",
    "            csv_reader = csv.reader(o)\n",
    "            header = True\n",
    "            for row in csv_reader:\n",
    "                if header:\n",
    "                    header = False\n",
    "                else:\n",
    "                    print(filename, row[5])\n",
    "                    df = pd.read_csv(filename)\n",
    "                    print(df.shape)\n",
    "                    break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(107, 8)"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"out/0915/filtered/edges/ENCODE CCRE ACTIVITY.regulates.ENTREZ.edges.csv\")\n",
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['ENSEMBL:ENSG00000274253 CUI'], dtype=object)"
      ]
     },
     "execution_count": 70,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.target.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.regulates.ENTREZ.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.isa.ENCODE CCRE H3K27AC.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.has part.Tissue.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.molecularly interacts with.ENCODE RBS HEPG2.edges.csv ERCC\n",
      "out/0915/filtered/edges/CLINGEN ALLELE REGISTRY.located in.ENCODE CCRE.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS HEPG2 K562.overlaps.Gene.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS 150 NO OVERLAP.is subsequence of.ENCODE RBS HEPG2 K562.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.has part.ENCODE CCRE.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.has part.Tissue.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.molecularly interacts with.ENCODE RBS HEPG2 K562.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.has part.Body Part, Organ, or Organ Component.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.positively regulated by.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.isa.ENCODE CCRE H3K4ME3.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS K562.overlaps.Gene.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.ENSEMBL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE.location of.CLINGEN ALLELE REGISTRY.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENTREZ.positively regulated by.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/Tissue.part of.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.regulates.ENSEMBL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENSEMBL.positively regulated by.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.has part.Body Part, Organ, or Organ Component.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.not predicted in.Body Substance.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS HEPG2.overlaps.Gene.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS HEPG2 K562.has subsequence.ENCODE RBS 150 NO OVERLAP.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.negatively regulates.Gene.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS 150 NO OVERLAP.is subsequence of.ENCODE RBS HEPG2.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.isa.ENCODE CCRE CTCF.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS 150 NO OVERLAP.overlaps.Gene.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENSEMBL.regulated by.ENCODE CCRE ACTIVITY.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.molecularly interacts with.ENCODE RBS K562.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS 150 NO OVERLAP.is subsequence of.ENCODE RBS K562.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS K562.overlaps.ENSEMBL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENTREZ.regulated by.ENCODE CCRE ACTIVITY.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.has part.CLINGEN ALLELE REGISTRY.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENSEMBL.negatively regulated by.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/Body Part, Organ, or Organ Component.part of.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.predicted in.Body Substance.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS HEPG2.has subsequence.ENCODE RBS 150 NO OVERLAP.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS HEPG2 K562.overlaps.ENSEMBL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS 150 NO OVERLAP.correlated in.Body Substance.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS 150 NO OVERLAP.overlaps.ENSEMBL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS HEPG2.overlaps.ENSEMBL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.regulates.Gene.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.negatively regulates.ENSEMBL.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.ENTREZ.edges.csv ERCC\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.Gene.edges.csv ERCC\n",
      "out/0915/filtered/edges/Tissue.part of.ENCODE CCRE ACTIVITY.edges.csv ERCC\n",
      "out/0915/filtered/edges/Body Part, Organ, or Organ Component.part of.ENCODE CCRE ACTIVITY.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS 150 NO OVERLAP.not correlated in.Body Substance.edges.csv ERCC\n",
      "out/0915/filtered/edges/CLINGEN ALLELE REGISTRY.part of.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE RBS K562.has subsequence.ENCODE RBS 150 NO OVERLAP.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.negatively regulated by.GTEXEQTL.edges.csv ERCC\n",
      "out/0915/filtered/edges/Gene.regulated by.ENCODE CCRE ACTIVITY.edges.csv ERCC\n",
      "out/0915/filtered/edges/ENCODE CCRE.part of.ENCODE CCRE ACTIVITY.edges.csv ERCC\n"
     ]
    }
   ],
   "source": [
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    with open(filename) as o:\n",
    "        csv_reader = csv.reader(o)\n",
    "        header = True\n",
    "        for row in csv_reader:\n",
    "            if header:\n",
    "                header = False\n",
    "            else:\n",
    "                if 'ERCC' == row[-1]:\n",
    "                    print(filename, row[-1])\n",
    "                break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>ENSEMBL</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>ENSEMBL:ENSG00000274253 CUI</th>\n",
       "      <td>ENSG00000274253</td>\n",
       "      <td>ENTREZ</td>\n",
       "      <td>ENSEMBL:ENSG00000274253</td>\n",
       "      <td>6.0</td>\n",
       "      <td>22757835.0</td>\n",
       "      <td>22778741.0</td>\n",
       "      <td>ENTREZ::283683</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       label    type                  ENSEMBL  \\\n",
       "id                                                                              \n",
       "ENSEMBL:ENSG00000274253 CUI  ENSG00000274253  ENTREZ  ENSEMBL:ENSG00000274253   \n",
       "\n",
       "                             ENSEMBL value  ENSEMBL lowerbound  \\\n",
       "id                                                               \n",
       "ENSEMBL:ENSG00000274253 CUI            6.0          22757835.0   \n",
       "\n",
       "                             ENSEMBL upperbound          ENTREZ  \n",
       "id                                                               \n",
       "ENSEMBL:ENSG00000274253 CUI          22778741.0  ENTREZ::283683  "
      ]
     },
     "execution_count": 163,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Replace ENTREZ node to gene\n",
    "entrez = pd.read_csv(\"out/0915/filtered/nodes/ENTREZ.nodes.csv\", index_col=0)\n",
    "entrez.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 164,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_30234/4247972648.py:1: DtypeWarning: Columns (15,16,19) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  gene_df = pd.read_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\", index_col=0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>LCH_NW</th>\n",
       "      <th>MSH</th>\n",
       "      <th>CSP</th>\n",
       "      <th>SNOMEDCT_US</th>\n",
       "      <th>HGNC</th>\n",
       "      <th>MTH</th>\n",
       "      <th>OMIM</th>\n",
       "      <th>...</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "      <th>ENTREZ lowerbound</th>\n",
       "      <th>ENTREZ upperbound</th>\n",
       "      <th>PUBCHEM</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "      <th>EC ID</th>\n",
       "      <th>is_Enzyme</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1420882</th>\n",
       "      <td>TPT1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C101389</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12022</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:600763</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341284.0</td>\n",
       "      <td>ENTREZ:7178</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341183.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P13693</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1412662</th>\n",
       "      <td>ATP5MC1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:841</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:603192</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>ENTREZ:516</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P05496</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1824623</th>\n",
       "      <td>CCDC96 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:26900</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:619347</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>ENTREZ:257236</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q2M329</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C3543374</th>\n",
       "      <td>RN7SL531P gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:46547</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43492032.0</td>\n",
       "      <td>43492313.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1336927</th>\n",
       "      <td>VAV3 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C24894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12659</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:605541</td>\n",
       "      <td>...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>ENTREZ:10451</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9UKW4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   label  type          NCI LCH_NW  MSH  CSP SNOMEDCT_US  \\\n",
       "id                                                                         \n",
       "C1420882       TPT1 gene  Gene  NCI:C101389    NaN  NaN  NaN         NaN   \n",
       "C1412662    ATP5MC1 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1824623     CCDC96 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C3543374  RN7SL531P gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1336927       VAV3 gene  Gene   NCI:C24894    NaN  NaN  NaN         NaN   \n",
       "\n",
       "                HGNC         MTH         OMIM  ... ENSEMBL value  \\\n",
       "id                                             ...                 \n",
       "C1420882  HGNC:12022  MTH:NOCODE  OMIM:600763  ...          17.0   \n",
       "C1412662    HGNC:841  MTH:NOCODE  OMIM:603192  ...          14.0   \n",
       "C1824623  HGNC:26900  MTH:NOCODE  OMIM:619347  ...           6.0   \n",
       "C3543374  HGNC:46547  MTH:NOCODE          NaN  ...           2.0   \n",
       "C1336927  HGNC:12659  MTH:NOCODE  OMIM:605541  ...          16.0   \n",
       "\n",
       "         ENSEMBL lowerbound ENSEMBL upperbound         ENTREZ  \\\n",
       "id                                                              \n",
       "C1420882         45333471.0         45341284.0    ENTREZ:7178   \n",
       "C1412662         48892765.0         48895871.0     ENTREZ:516   \n",
       "C1824623          7040849.0          7043001.0  ENTREZ:257236   \n",
       "C3543374         43492032.0         43492313.0            NaN   \n",
       "C1336927        107571161.0        107965180.0   ENTREZ:10451   \n",
       "\n",
       "         ENTREZ lowerbound ENTREZ upperbound PUBCHEM UNIPROTKB EC ID is_Enzyme  \n",
       "id                                                                              \n",
       "C1420882        45333471.0        45341183.0     NaN    P13693   NaN     False  \n",
       "C1412662        48892765.0        48895871.0     NaN    P05496   NaN     False  \n",
       "C1824623         7040849.0         7043001.0     NaN    Q2M329   NaN     False  \n",
       "C3543374               NaN               NaN     NaN       NaN   NaN     False  \n",
       "C1336927       107571161.0       107965180.0     NaN    Q9UKW4   NaN     False  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 164,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gene_df = pd.read_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\", index_col=0)\n",
    "gene_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.loc[\"ENSEMBL:ENSG00000274253 CUI\"] = entrez.loc[\"ENSEMBL:ENSG00000274253 CUI\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.type = \"Gene\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 168,
   "metadata": {},
   "outputs": [],
   "source": [
    "gene_df.to_csv(\"out/0915/filtered/nodes/Gene.nodes.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 167,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.remove('out/0915/filtered/nodes/ENTREZ.nodes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 170,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Drug.positively regulates.Gene.edges.csv\n",
      "out/0915/filtered/edges/ENCODE CCRE ACTIVITY.regulates.Gene.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively regulated by.Drug.edges.csv\n",
      "out/0915/filtered/edges/Gene.positively regulated by.GTEXEQTL.edges.csv\n",
      "out/0915/filtered/edges/Gene.regulated by.ENCODE CCRE ACTIVITY.edges.csv\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.Gene.edges.csv\n"
     ]
    }
   ],
   "source": [
    "for filename in sorted(glob('out/0915/filtered/edges/*')):\n",
    "    if 'ENTREZ.' in filename:\n",
    "        append_file = filename.replace(\"ENTREZ\", \"Gene\")\n",
    "        print(append_file)\n",
    "        with open(append_file, \"a\") as a:\n",
    "            csv_writer = csv.writer(a)\n",
    "            with open(filename) as o:\n",
    "                csv_reader = csv.reader(o)\n",
    "                h = True\n",
    "                for row in csv_reader:\n",
    "                    if h:\n",
    "                        h = False\n",
    "                    else:\n",
    "                        if not row[5] == \"LINCS\":\n",
    "                            csv_writer.writerow(row)\n",
    "        os.remove(filename)   \n",
    "                    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 178,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/GTEXEQTL.has part.Tissue.edges.csv ['ERCC']\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Body Part, Organ, or Organ Component.edges.csv ['GTEX']\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.ENSEMBL.edges.csv ['ERCC']\n",
      "out/0915/filtered/edges/GTEXEQTL.has part.Body Part, Organ, or Organ Component.edges.csv ['ERCC']\n",
      "out/0915/filtered/edges/GTEXEQTL.negatively regulates.Gene.edges.csv ['ERCC']\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Cell.edges.csv ['GTEX']\n",
      "out/0915/filtered/edges/GTEXEQTL.p value.PVALUEBINS.edges.csv ['GTEX']\n",
      "out/0915/filtered/edges/GTEXEQTL.has part.CLINGEN ALLELE REGISTRY.edges.csv ['ERCC']\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Gene.edges.csv ['GTEX']\n",
      "out/0915/filtered/edges/GTEXEQTL.negatively regulates.ENSEMBL.edges.csv ['ERCC']\n",
      "out/0915/filtered/edges/GTEXEQTL.positively regulates.Gene.edges.csv ['ERCC']\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.CHLO.edges.csv ['GTEX']\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.Tissue.edges.csv ['GTEX']\n",
      "out/0915/filtered/edges/GTEXEQTL.located in.EFO.edges.csv ['GTEX']\n"
     ]
    }
   ],
   "source": [
    "gtex = \"GTEXEQTL\"\n",
    "for filename in glob('out/0915/filtered/edges/*'):\n",
    "    # relations.add(match['relation'])\n",
    "    match = re.match(edge_pattern, filename).groupdict()\n",
    "    if gtex in match[\"source_type\"]:\n",
    "        df = pd.read_csv(filename)\n",
    "        print(filename, df.DCC.unique())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>GTEXEQTL:eQTL.chr7.64957448.C.A.b38.Esophagus....</td>\n",
       "      <td>located_in</td>\n",
       "      <td>C2681341</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CCT6P3 gene</td>\n",
       "      <td>GTEXEQTL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>GTEXEQTL:eQTL.chr6.32603032.A.G.b38.Brain.Hipp...</td>\n",
       "      <td>located_in</td>\n",
       "      <td>C1415580</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HLA-DRB5 gene</td>\n",
       "      <td>GTEXEQTL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>GTEXEQTL:eQTL.chr20.35712534.C.T.b38.Adrenal.G...</td>\n",
       "      <td>located_in</td>\n",
       "      <td>C1413675</td>\n",
       "      <td>NaN</td>\n",
       "      <td>CPNE1 gene</td>\n",
       "      <td>GTEXEQTL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>GTEXEQTL:eQTL.chr2.61320500.T.A.b38.Brain.Fron...</td>\n",
       "      <td>located_in</td>\n",
       "      <td>C2681200</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C2orf74 gene</td>\n",
       "      <td>GTEXEQTL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>GTEXEQTL:eQTL.chr7.72725280.G.T.b38.Testis CUI</td>\n",
       "      <td>located_in</td>\n",
       "      <td>C2240136</td>\n",
       "      <td>NaN</td>\n",
       "      <td>TYW1B gene</td>\n",
       "      <td>GTEXEQTL</td>\n",
       "      <td>NaN</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                              source    relation    target  \\\n",
       "0  GTEXEQTL:eQTL.chr7.64957448.C.A.b38.Esophagus....  located_in  C2681341   \n",
       "1  GTEXEQTL:eQTL.chr6.32603032.A.G.b38.Brain.Hipp...  located_in  C1415580   \n",
       "2  GTEXEQTL:eQTL.chr20.35712534.C.T.b38.Adrenal.G...  located_in  C1413675   \n",
       "3  GTEXEQTL:eQTL.chr2.61320500.T.A.b38.Brain.Fron...  located_in  C2681200   \n",
       "4     GTEXEQTL:eQTL.chr7.72725280.G.T.b38.Testis CUI  located_in  C2240136   \n",
       "\n",
       "   source_label   target_label       SAB  evidence   DCC  \n",
       "0           NaN    CCT6P3 gene  GTEXEQTL       NaN  GTEX  \n",
       "1           NaN  HLA-DRB5 gene  GTEXEQTL       NaN  GTEX  \n",
       "2           NaN     CPNE1 gene  GTEXEQTL       NaN  GTEX  \n",
       "3           NaN   C2orf74 gene  GTEXEQTL       NaN  GTEX  \n",
       "4           NaN     TYW1B gene  GTEXEQTL       NaN  GTEX  "
      ]
     },
     "execution_count": 180,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"out/0915/filtered/edges/GTEXEQTL.located in.Gene.edges.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "import re\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>From</th>\n",
       "      <th>To</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>A0A087X1C5</td>\n",
       "      <td>HGNC:2624</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>A0A0B4J2F0</td>\n",
       "      <td>HGNC:50696</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>A0A0C5B5G6</td>\n",
       "      <td>HGNC:7470</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>A0A0K2S4Q6</td>\n",
       "      <td>HGNC:52292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>A0A0U1RRE5</td>\n",
       "      <td>HGNC:50713</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         From          To\n",
       "0  A0A087X1C5   HGNC:2624\n",
       "1  A0A0B4J2F0  HGNC:50696\n",
       "2  A0A0C5B5G6   HGNC:7470\n",
       "3  A0A0K2S4Q6  HGNC:52292\n",
       "4  A0A0U1RRE5  HGNC:50713"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "id_mapping = pd.read_csv('dd_data/idmapping_2023_08_24.tsv.gz', sep='\\t')\n",
    "id_mapping.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "mapper = {}\n",
    "for i, row in id_mapping.iterrows():\n",
    "    f = row[\"From\"]\n",
    "    t = row[\"To\"]\n",
    "    mapper[\"UNIPROTKB:%s CUI\"%f] = t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/var/folders/wq/7bkxhy8j7ql194c1fbby4gy40000gn/T/ipykernel_55701/1599047433.py:1: DtypeWarning: Columns (15,16,19) have mixed types. Specify dtype option on import or set low_memory=False.\n",
      "  genes = pd.read_csv('out/0915/filtered/nodes/Gene.nodes.csv', index_col=0)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>type</th>\n",
       "      <th>NCI</th>\n",
       "      <th>LCH_NW</th>\n",
       "      <th>MSH</th>\n",
       "      <th>CSP</th>\n",
       "      <th>SNOMEDCT_US</th>\n",
       "      <th>HGNC</th>\n",
       "      <th>MTH</th>\n",
       "      <th>OMIM</th>\n",
       "      <th>...</th>\n",
       "      <th>ENSEMBL value</th>\n",
       "      <th>ENSEMBL lowerbound</th>\n",
       "      <th>ENSEMBL upperbound</th>\n",
       "      <th>ENTREZ</th>\n",
       "      <th>ENTREZ lowerbound</th>\n",
       "      <th>ENTREZ upperbound</th>\n",
       "      <th>PUBCHEM</th>\n",
       "      <th>UNIPROTKB</th>\n",
       "      <th>EC ID</th>\n",
       "      <th>is_Enzyme</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C1420882</th>\n",
       "      <td>TPT1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C101389</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12022</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:600763</td>\n",
       "      <td>...</td>\n",
       "      <td>17.0</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341284.0</td>\n",
       "      <td>ENTREZ:7178</td>\n",
       "      <td>45333471.0</td>\n",
       "      <td>45341183.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P13693</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1412662</th>\n",
       "      <td>ATP5MC1 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:841</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:603192</td>\n",
       "      <td>...</td>\n",
       "      <td>14.0</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>ENTREZ:516</td>\n",
       "      <td>48892765.0</td>\n",
       "      <td>48895871.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>P05496</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1824623</th>\n",
       "      <td>CCDC96 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:26900</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:619347</td>\n",
       "      <td>...</td>\n",
       "      <td>6.0</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>ENTREZ:257236</td>\n",
       "      <td>7040849.0</td>\n",
       "      <td>7043001.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q2M329</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C3543374</th>\n",
       "      <td>RN7SL531P gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:46547</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>2.0</td>\n",
       "      <td>43492032.0</td>\n",
       "      <td>43492313.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C1336927</th>\n",
       "      <td>VAV3 gene</td>\n",
       "      <td>Gene</td>\n",
       "      <td>NCI:C24894</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>HGNC:12659</td>\n",
       "      <td>MTH:NOCODE</td>\n",
       "      <td>OMIM:605541</td>\n",
       "      <td>...</td>\n",
       "      <td>16.0</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>ENTREZ:10451</td>\n",
       "      <td>107571161.0</td>\n",
       "      <td>107965180.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>Q9UKW4</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 30 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                   label  type          NCI LCH_NW  MSH  CSP SNOMEDCT_US  \\\n",
       "id                                                                         \n",
       "C1420882       TPT1 gene  Gene  NCI:C101389    NaN  NaN  NaN         NaN   \n",
       "C1412662    ATP5MC1 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1824623     CCDC96 gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C3543374  RN7SL531P gene  Gene          NaN    NaN  NaN  NaN         NaN   \n",
       "C1336927       VAV3 gene  Gene   NCI:C24894    NaN  NaN  NaN         NaN   \n",
       "\n",
       "                HGNC         MTH         OMIM  ... ENSEMBL value  \\\n",
       "id                                             ...                 \n",
       "C1420882  HGNC:12022  MTH:NOCODE  OMIM:600763  ...          17.0   \n",
       "C1412662    HGNC:841  MTH:NOCODE  OMIM:603192  ...          14.0   \n",
       "C1824623  HGNC:26900  MTH:NOCODE  OMIM:619347  ...           6.0   \n",
       "C3543374  HGNC:46547  MTH:NOCODE          NaN  ...           2.0   \n",
       "C1336927  HGNC:12659  MTH:NOCODE  OMIM:605541  ...          16.0   \n",
       "\n",
       "         ENSEMBL lowerbound ENSEMBL upperbound         ENTREZ  \\\n",
       "id                                                              \n",
       "C1420882         45333471.0         45341284.0    ENTREZ:7178   \n",
       "C1412662         48892765.0         48895871.0     ENTREZ:516   \n",
       "C1824623          7040849.0          7043001.0  ENTREZ:257236   \n",
       "C3543374         43492032.0         43492313.0            NaN   \n",
       "C1336927        107571161.0        107965180.0   ENTREZ:10451   \n",
       "\n",
       "         ENTREZ lowerbound ENTREZ upperbound PUBCHEM UNIPROTKB EC ID is_Enzyme  \n",
       "id                                                                              \n",
       "C1420882        45333471.0        45341183.0     NaN    P13693   NaN     False  \n",
       "C1412662        48892765.0        48895871.0     NaN    P05496   NaN     False  \n",
       "C1824623         7040849.0         7043001.0     NaN    Q2M329   NaN     False  \n",
       "C3543374               NaN               NaN     NaN       NaN   NaN     False  \n",
       "C1336927       107571161.0       107965180.0     NaN    Q9UKW4   NaN     False  \n",
       "\n",
       "[5 rows x 30 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "genes = pd.read_csv('out/0915/filtered/nodes/Gene.nodes.csv', index_col=0)\n",
    "genes.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "22634"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "hgnc_to_cui = {}\n",
    "uniprot_to_cui = {}\n",
    "for i, row in genes.iterrows():\n",
    "    hgnc = row[\"HGNC\"]\n",
    "    uniprot = row[\"UNIPROTKB\"]\n",
    "    if hgnc and type(hgnc) == str:\n",
    "        hgnc_to_cui[hgnc] = i\n",
    "    if uniprot:\n",
    "        uniprot_to_cui[\"UNIPROTKB:%s CUI\"%uniprot] = i\n",
    "len(uniprot_to_cui)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2592 2546 2546\n"
     ]
    }
   ],
   "source": [
    "matched = []\n",
    "counter = 0\n",
    "hgnc_match = []\n",
    "for i in genes.index:\n",
    "    if i in mapper:\n",
    "        matched.append(i)\n",
    "        if mapper[i] in hgnc_to_cui:\n",
    "            hgnc_match.append((i, mapper[i], hgnc_to_cui[mapper[i]]))\n",
    "    if \"UNIPROTKB\" in i:\n",
    "        counter += 1\n",
    "print(counter, len(matched), len(hgnc_match))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "genes = genes.drop(labels=[i[0] for i in hgnc_match])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "label_mapper = {}\n",
    "for i in hgnc_match:\n",
    "    ind = i[2]\n",
    "    label = genes.at[ind, 'label']\n",
    "    label_mapper[ind] = label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('CAPN2 gene', 'C1413112')"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_mapper[ind], ind"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'(?P<directory>.+)/(?P<source_type>.+)\\\\.(?P<relation>.+)\\\\.(?P<target_type>.+)\\\\.(?P<entity>.+)\\\\.csv'"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "edge_pattern"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "with_uniprot = set()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|▏         | 3/183 [00:25<28:05,  9.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Gene.molecularly interacts with.ENCODE RBS HEPG2.edges.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  7%|▋         | 13/183 [00:56<08:06,  2.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Drug.bioactivity.Gene.edges.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 13%|█▎        | 24/183 [01:12<03:23,  1.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Gene.molecularly interacts with.ENCODE RBS HEPG2 K562.edges.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|███▍      | 64/183 [05:43<08:11,  4.13s/it]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Gene.not predicted in.Body Substance.edges.csv\n",
      "out/0915/filtered/edges/GLYGEN GLYCOSYLATION.has enzyme protein.Gene.edges.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|█████     | 92/183 [06:34<03:27,  2.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/GLYCOSYLTRANSFERASE REACTION.has enzyme protein.Gene.edges.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 53%|█████▎    | 97/183 [06:44<02:56,  2.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Gene.molecularly interacts with.ENCODE RBS K562.edges.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|██████▎   | 115/183 [07:30<03:14,  2.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Gene.predicted in.Body Substance.edges.csv\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 183/183 [11:15<00:00,  3.69s/it]\n"
     ]
    }
   ],
   "source": [
    "for filename in tqdm(glob(\"out/0915/filtered/edges/*Gene.*.csv\")):\n",
    "    match = re.match(edge_pattern, filename).groupdict()\n",
    "    column = \"\" \n",
    "    if match[\"target_type\"] == \"Gene\":\n",
    "        column = \"target\"\n",
    "    elif match[\"source_type\"] == \"Gene\":\n",
    "        column = \"source\"\n",
    "    df = pd.read_csv(filename)\n",
    "    save = False\n",
    "    for i, row in df.iterrows():\n",
    "        gene_id = row[column]\n",
    "        if gene_id in mapper:\n",
    "            save = True\n",
    "            hgnc = mapper[gene_id]\n",
    "            cui = hgnc_to_cui[mapper[gene_id]]\n",
    "            label = label_mapper[cui]\n",
    "            df.at[i, column] = cui\n",
    "            df.at[i, \"%s_label\"%column] = cui\n",
    "    if save:\n",
    "        print(filename)\n",
    "        df.to_csv(filename, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'AKAP1 gene'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_mapper['C1412311']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "source                                            C2681866\n",
       "relation                      gene_associated_with_pathway\n",
       "target                    MSIGDB_SYSTEMATIC_NAME_M4235 CUI\n",
       "source_label                                    NEAT1 gene\n",
       "target_label    GRAHAM_CML_DIVIDING_VS_NORMAL_QUIESCENT_DN\n",
       "SAB                                                 MSIGDB\n",
       "evidence                                               NaN\n",
       "DCC                                                 MSIGDB\n",
       "Name: 148414, dtype: object"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "genes.to_csv('out/0915/filtered/nodes/Gene.nodes.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "out/0915/filtered/edges/Body Part, Organ, or Organ Component.expresses.Gene.edges.csv\n",
      "36\n",
      "out/0915/filtered/edges/Tissue.expresses.Gene.edges.csv\n",
      "6\n",
      "out/0915/filtered/edges/Body Location or Region.expresses.Gene.edges.csv\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "for i in glob(\"out/0915/filtered/edges/*expresses*\"):\n",
    "    print(i)\n",
    "    df = pd.read_csv(i)\n",
    "    print(len(df.source_label.unique()))\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>C0001625</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Adrenal Glands</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>C0003956</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Ascending aorta structure</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>C0205042</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Coronary artery</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>C0085427</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Tibial Arteries</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>C0005682</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C2239334</td>\n",
       "      <td>Urinary Bladder</td>\n",
       "      <td>DDX11L1 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>0.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     source   relation    target               source_label  target_label  \\\n",
       "0  C0001625  expresses  C2239334             Adrenal Glands  DDX11L1 gene   \n",
       "1  C0003956  expresses  C2239334  Ascending aorta structure  DDX11L1 gene   \n",
       "2  C0205042  expresses  C2239334            Coronary artery  DDX11L1 gene   \n",
       "3  C0085427  expresses  C2239334            Tibial Arteries  DDX11L1 gene   \n",
       "4  C0005682  expresses  C2239334            Urinary Bladder  DDX11L1 gene   \n",
       "\n",
       "       SAB  evidence   DCC  \n",
       "0  GTEXEXP       0.0  GTEX  \n",
       "1  GTEXEXP       0.0  GTEX  \n",
       "2  GTEXEXP       0.0  GTEX  \n",
       "3  GTEXEXP       0.0  GTEX  \n",
       "4  GTEXEXP       0.0  GTEX  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(\"out/0915/filtered/edges/Body Part, Organ, or Organ Component.expresses.Gene.edges.csv\")\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array(['Adrenal Glands', 'Ascending aorta structure', 'Coronary artery',\n",
       "       'Tibial Arteries', 'Urinary Bladder', 'Amygdaloid structure',\n",
       "       'Cerebellum', 'Cortex of frontal lobe', 'Hippocampus (Brain)',\n",
       "       'Hypothalamic structure', 'Substantia nigra structure',\n",
       "       'Exocervix', 'Endocervix', 'Sigmoid Flexure', 'Transverse colon',\n",
       "       'Inferior esophageal sphincter structure',\n",
       "       'Lamina muscularis of esophageal mucous membrane',\n",
       "       'Fallopian Tubes', 'Right auricular appendage',\n",
       "       'Myocardium of left ventricle', 'Structure of cortex of kidney',\n",
       "       'Right lobe of liver', 'Structure of left upper lobe of lung',\n",
       "       'Structure of medial head of gastrocnemius muscle',\n",
       "       'Structure of tibial nerve', 'Ovary', 'Body of pancreas',\n",
       "       'Pituitary Gland', 'Prostate',\n",
       "       'Structure of aggregated lymphoid follicle of small intestine',\n",
       "       'Spleen', 'Stomach', 'Testis', 'Thyroid Gland', 'Uterus', 'Vagina'],\n",
       "      dtype=object)"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.source_label.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from glob import glob\n",
    "from tqdm import tqdm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>source</th>\n",
       "      <th>relation</th>\n",
       "      <th>target</th>\n",
       "      <th>source_label</th>\n",
       "      <th>target_label</th>\n",
       "      <th>SAB</th>\n",
       "      <th>evidence</th>\n",
       "      <th>DCC</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>129016</th>\n",
       "      <td>C2338243</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C1415481</td>\n",
       "      <td>Portion of venous blood</td>\n",
       "      <td>HBB gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>177050</th>\n",
       "      <td>C2338243</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C1415478</td>\n",
       "      <td>Portion of venous blood</td>\n",
       "      <td>HBA2 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13866</th>\n",
       "      <td>C2338243</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C1335798</td>\n",
       "      <td>Portion of venous blood</td>\n",
       "      <td>S100A9 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>244763</th>\n",
       "      <td>UBERON:0008367 CUI</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C1537992</td>\n",
       "      <td>breast epithelium</td>\n",
       "      <td>MT-ND4 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>58322</th>\n",
       "      <td>C0930558</td>\n",
       "      <td>expresses</td>\n",
       "      <td>C1417500</td>\n",
       "      <td>anterior lingual gland</td>\n",
       "      <td>MUC7 gene</td>\n",
       "      <td>GTEXEXP</td>\n",
       "      <td>40000.0</td>\n",
       "      <td>GTEX</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    source   relation    target             source_label  \\\n",
       "129016            C2338243  expresses  C1415481  Portion of venous blood   \n",
       "177050            C2338243  expresses  C1415478  Portion of venous blood   \n",
       "13866             C2338243  expresses  C1335798  Portion of venous blood   \n",
       "244763  UBERON:0008367 CUI  expresses  C1537992        breast epithelium   \n",
       "58322             C0930558  expresses  C1417500   anterior lingual gland   \n",
       "\n",
       "       target_label      SAB  evidence   DCC  \n",
       "129016     HBB gene  GTEXEXP  300000.0  GTEX  \n",
       "177050    HBA2 gene  GTEXEXP   90000.0  GTEX  \n",
       "13866   S100A9 gene  GTEXEXP   40000.0  GTEX  \n",
       "244763  MT-ND4 gene  GTEXEXP   40000.0  GTEX  \n",
       "58322     MUC7 gene  GTEXEXP   40000.0  GTEX  "
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.read_csv(\"out/0915/filtered/edges/Tissue.expresses.Gene.edges.csv\").sort_values(\"evidence\", ascending=False).head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 643/643 [01:50<00:00,  5.83it/s]\n"
     ]
    }
   ],
   "source": [
    "dccs = {}\n",
    "for i in tqdm(glob('out/0915/filtered/edges/*')):\n",
    "    df = pd.read_csv(i)\n",
    "    for k,v in df.groupby([\"DCC\"])[\"DCC\"].count().items():\n",
    "        if k not in dccs:\n",
    "            dccs[k] = 0\n",
    "        dccs[k] += v\n",
    "    with open(\"output/ids/%s\"%df.DCC.unique()[0], \"a\") as o:\n",
    "        o.write(\"\\n\".join(list(set(list(df.source) + list(df.target)))))\n",
    "        o.write(\"\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "for k,v in dccs.items():\n",
    "    with open(\"output/ids/%s\"%k) as o:\n",
    "        nodes = len(set(o.read().strip().split(\"\\n\")))\n",
    "        dccs[k] = {\n",
    "            \"nodes\": nodes,\n",
    "            \"edges\": v[\"edges\"]\n",
    "        }\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>nodes</th>\n",
       "      <th>edges</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>MW</th>\n",
       "      <td>11030</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 116549}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HGNCHPO</th>\n",
       "      <td>14300</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 655074}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GTEX</th>\n",
       "      <td>2862091</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 36369763}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>IDG</th>\n",
       "      <td>329023</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 435863}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HCOPMP</th>\n",
       "      <td>27631</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 233820}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>ERCC</th>\n",
       "      <td>4072618</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 27891997}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>CLINVAR</th>\n",
       "      <td>20105</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 53275}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>KF</th>\n",
       "      <td>145798</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 234625}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>HPOMP</th>\n",
       "      <td>1176</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 1214}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MSIGDB</th>\n",
       "      <td>44901</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 1305281}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GLYGEN</th>\n",
       "      <td>182445</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 464684}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>LINCS</th>\n",
       "      <td>8941</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 471797}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4DN</th>\n",
       "      <td>766621</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 2632949}</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>MOTORPAC</th>\n",
       "      <td>14495</td>\n",
       "      <td>{'nodes': 4072618, 'edges': 34284}</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "            nodes                                  edges\n",
       "MW          11030    {'nodes': 4072618, 'edges': 116549}\n",
       "HGNCHPO     14300    {'nodes': 4072618, 'edges': 655074}\n",
       "GTEX      2862091  {'nodes': 4072618, 'edges': 36369763}\n",
       "IDG        329023    {'nodes': 4072618, 'edges': 435863}\n",
       "HCOPMP      27631    {'nodes': 4072618, 'edges': 233820}\n",
       "ERCC      4072618  {'nodes': 4072618, 'edges': 27891997}\n",
       "CLINVAR     20105     {'nodes': 4072618, 'edges': 53275}\n",
       "KF         145798    {'nodes': 4072618, 'edges': 234625}\n",
       "HPOMP        1176      {'nodes': 4072618, 'edges': 1214}\n",
       "MSIGDB      44901   {'nodes': 4072618, 'edges': 1305281}\n",
       "GLYGEN     182445    {'nodes': 4072618, 'edges': 464684}\n",
       "LINCS        8941    {'nodes': 4072618, 'edges': 471797}\n",
       "4DN        766621   {'nodes': 4072618, 'edges': 2632949}\n",
       "MOTORPAC    14495     {'nodes': 4072618, 'edges': 34284}"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame.from_dict(dccs, orient=\"index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "orig_nbformat": 4
 },
 "nbformat": 4,
 "nbformat_minor": 2
}