ukbb_parser demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Setups"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import sys\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from IPython.display import display"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from ukbb_parser import create_dataset, create_ICD10_dataset, get_chrom_raw_marker_data, get_chrom_imputation_data\n",
    "from ukbb_parser.shared_utils.util import summarize"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# XXX: DELTE THIS CELL (and DO NOT execute it)!\n",
    "\n",
    "# We override the 'summarize' function so that this notebook could be published online, together with its output, without\n",
    "# compormising the privacy of the UKBB's participants. When this version is used, all displayed dataframes and series\n",
    "# values will be shuffled (column-wise) using a cryptographically secure random generator.\n",
    "# When using this notebook as part of your real research, you clearly do not want this shuffling (which would completely\n",
    "# obscure the true values in your analysis). Therefore you should delete this cell altogether (and no need to bother\n",
    "# yourself with this code).\n",
    "def summarize(df, n = 5):\n",
    "    \n",
    "    import secrets\n",
    "    \n",
    "    if isinstance(df, pd.DataFrame) and df.columns.tolist() == ['coding', 'meaning', 'parent_id', 'selectable', \\\n",
    "            'children_ids', 'samples', 'n_samples']:\n",
    "        \n",
    "        # When summarizing ICD10_tree (see below), it is sufficient to replace the samples with random numbers.\n",
    "        displayed_head = df.head(n).copy()\n",
    "        displayed_head['samples'] = displayed_head['n_samples'].apply(lambda n_samples: set(np.random.randint(0, \\\n",
    "                200000, n_samples)))\n",
    "        display(displayed_head)\n",
    "        print('Displaying %d of %d records (replaced the samples with random indices).' % (n, len(df)))\n",
    "        return\n",
    "    \n",
    "    if isinstance(df, pd.DataFrame) and df.columns.tolist() == ['fid', 'iid', 'father', 'mother', 'gender', 'trait', 'i']:\n",
    "        # When displaying a fam dataframe, we do not want to shuffle the 'i' values.\n",
    "        cols_to_exclude_from_shuffling = {'i'}\n",
    "    else:\n",
    "        cols_to_exclude_from_shuffling = set()\n",
    "    \n",
    "    def _sample_series(original_series):\n",
    "        return pd.Series([secrets.choice(original_series.values) for _ in range(n)]).rename(original_series.name)\n",
    "    \n",
    "    if isinstance(df, pd.Series):\n",
    "        displayed_head = _sample_series(df)\n",
    "    elif isinstance(df, pd.DataFrame):\n",
    "        displayed_head = pd.concat([df[col_name].head(n).reset_index(drop = True) if col_name in \\\n",
    "                cols_to_exclude_from_shuffling else _sample_series(df[col_name]) for col_name in df.columns], axis = 1)\n",
    "    else:\n",
    "        raise TypeError('Unsupported type: %s' % type(df))\n",
    "    \n",
    "    display(displayed_head)\n",
    "    print('Displaying %d of %d records (shuffled).' % (n, len(df)))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Parsing phenotypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "PHENOTYPES = [\n",
    "    # The format is: ('field_name', field_id, 'field_type')\n",
    "    # field_name could be anything you like (this will determine the column name in the resulted dataframe)\n",
    "    # fiel_id is the ID of the UKBB field (e.g. http://biobank.ctsu.ox.ac.uk/crystal/field.cgi?id=50)\n",
    "    # field_type could be any of:\n",
    "    # - 'continuous' (will take the maximum among all the existing records for each sample)\n",
    "    # - 'binary' (i.e. 0 or 1)\n",
    "    # - 'set' (will return a set with all the values found for each sample; note that this is dramatically slower to parse)\n",
    "    # - You can also specify a Python function that takes all the values (as a dataframe) and returns the final values (as\n",
    "    # a series with the same index).\n",
    "    ('height', 50, 'continuous'),\n",
    "    ('diastolic_blood_preasure', 4079, 'continuous'),\n",
    "    ('red_blood_cell_count', 30010, 'continuous'),\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019_12_29-13:55:41] Reading all dataset rows of 26 columns (for 8 fields)...\n",
      "[2019_12_29-13:59:08] Finished after 0:03:27.152907.\n",
      "[2019_12_29-13:59:08] Read a dataset of 502520 samples.\n",
      "[2019_12_29-13:59:08] Knowing of 85 samples who have wished to withdraw, 0 of them are in the loaded dataset. Filtering out these records, the dataset has reduced from 502520 to 502520 samples.\n",
      "[2019_12_29-13:59:08] Of 502520 samples, 472638 are self-reported whites and 409624 are Caucasians according to their genetics. Keeping only the 409592 samples that are both.\n",
      "[2019_12_29-13:59:09] Parsing field sex...\n",
      "[2019_12_29-13:59:09] Parsing field year_of_birth...\n",
      "[2019_12_29-13:59:09] To avoid the \"dummy variable trap\", removing the AC_leeds column (38019 matching records).\n",
      "[2019_12_29-13:59:14] Filtering out 0 samples without genotyping metadata.\n",
      "[2019_12_29-13:59:14] Filtering out 312 samples with mismatching genetic and self-reported sex.\n",
      "[2019_12_29-13:59:16] To avoid the \"dummy variable trap\", removing the batch_UKBiLEVEAX_b2 column (4110 matching records).\n",
      "[2019_12_29-13:59:22] Constructed 60173 kinship groups (147731 samples), of which 52426 (128278 samples) are relevant for the dataset (i.e. containing at least 2 samples in the dataset). Picking only one representative of each group and removing the 75852 other samples in those groups has reduced the dataset from 409280 to 333428 samples.\n",
      "[2019_12_29-13:59:22] Parsing field height...\n",
      "[2019_12_29-13:59:22] Parsing field diastolic_blood_preasure...\n",
      "[2019_12_29-13:59:22] Parsing field red_blood_cell_count...\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    2992053\n",
       "1    1896950\n",
       "2    5486090\n",
       "3    1915867\n",
       "4    3573039\n",
       "Name: eid, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 333428 records (shuffled).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>height</th>\n",
       "      <th>diastolic_blood_preasure</th>\n",
       "      <th>red_blood_cell_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>173.0</td>\n",
       "      <td>87.0</td>\n",
       "      <td>4.881</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>170.0</td>\n",
       "      <td>82.0</td>\n",
       "      <td>4.728</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>163.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>4.415</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>168.0</td>\n",
       "      <td>71.0</td>\n",
       "      <td>4.429</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>160.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>3.751</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   height  diastolic_blood_preasure  red_blood_cell_count\n",
       "0   173.0                      87.0                 4.881\n",
       "1   170.0                      82.0                 4.728\n",
       "2   163.0                      71.0                 4.415\n",
       "3   168.0                      71.0                 4.429\n",
       "4   160.0                      91.0                 3.751"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 333428 records (shuffled).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>const</th>\n",
       "      <th>sex</th>\n",
       "      <th>year_of_birth</th>\n",
       "      <th>PC1</th>\n",
       "      <th>PC2</th>\n",
       "      <th>PC3</th>\n",
       "      <th>PC4</th>\n",
       "      <th>PC5</th>\n",
       "      <th>PC6</th>\n",
       "      <th>PC7</th>\n",
       "      <th>...</th>\n",
       "      <th>batch_UKBiLEVEAX_b1</th>\n",
       "      <th>batch_UKBiLEVEAX_b10</th>\n",
       "      <th>batch_UKBiLEVEAX_b11</th>\n",
       "      <th>batch_UKBiLEVEAX_b3</th>\n",
       "      <th>batch_UKBiLEVEAX_b4</th>\n",
       "      <th>batch_UKBiLEVEAX_b5</th>\n",
       "      <th>batch_UKBiLEVEAX_b6</th>\n",
       "      <th>batch_UKBiLEVEAX_b7</th>\n",
       "      <th>batch_UKBiLEVEAX_b8</th>\n",
       "      <th>batch_UKBiLEVEAX_b9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1945.0</td>\n",
       "      <td>-11.2472</td>\n",
       "      <td>3.19866</td>\n",
       "      <td>-3.274980</td>\n",
       "      <td>2.383010</td>\n",
       "      <td>-4.24618</td>\n",
       "      <td>0.041979</td>\n",
       "      <td>1.144430</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1941.0</td>\n",
       "      <td>-12.0387</td>\n",
       "      <td>1.86674</td>\n",
       "      <td>-2.405750</td>\n",
       "      <td>-0.424546</td>\n",
       "      <td>-9.16532</td>\n",
       "      <td>0.312106</td>\n",
       "      <td>1.413400</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>1942.0</td>\n",
       "      <td>-11.8184</td>\n",
       "      <td>4.80732</td>\n",
       "      <td>-0.006013</td>\n",
       "      <td>1.009260</td>\n",
       "      <td>-6.58821</td>\n",
       "      <td>-0.963682</td>\n",
       "      <td>0.589810</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1942.0</td>\n",
       "      <td>-12.4198</td>\n",
       "      <td>5.01533</td>\n",
       "      <td>-3.172800</td>\n",
       "      <td>-3.556280</td>\n",
       "      <td>-4.40342</td>\n",
       "      <td>-1.057110</td>\n",
       "      <td>0.997867</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1943.0</td>\n",
       "      <td>-11.3172</td>\n",
       "      <td>7.16993</td>\n",
       "      <td>-2.868510</td>\n",
       "      <td>-0.466165</td>\n",
       "      <td>-6.64145</td>\n",
       "      <td>-1.464440</td>\n",
       "      <td>-0.280808</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 173 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   const  sex  year_of_birth      PC1      PC2       PC3       PC4      PC5  \\\n",
       "0    1.0  1.0         1945.0 -11.2472  3.19866 -3.274980  2.383010 -4.24618   \n",
       "1    1.0  1.0         1941.0 -12.0387  1.86674 -2.405750 -0.424546 -9.16532   \n",
       "2    1.0  1.0         1942.0 -11.8184  4.80732 -0.006013  1.009260 -6.58821   \n",
       "3    1.0  0.0         1942.0 -12.4198  5.01533 -3.172800 -3.556280 -4.40342   \n",
       "4    1.0  0.0         1943.0 -11.3172  7.16993 -2.868510 -0.466165 -6.64145   \n",
       "\n",
       "        PC6       PC7  ...  batch_UKBiLEVEAX_b1  batch_UKBiLEVEAX_b10  \\\n",
       "0  0.041979  1.144430  ...                    0                     0   \n",
       "1  0.312106  1.413400  ...                    0                     0   \n",
       "2 -0.963682  0.589810  ...                    0                     0   \n",
       "3 -1.057110  0.997867  ...                    0                     0   \n",
       "4 -1.464440 -0.280808  ...                    0                     0   \n",
       "\n",
       "   batch_UKBiLEVEAX_b11  batch_UKBiLEVEAX_b3  batch_UKBiLEVEAX_b4  \\\n",
       "0                     0                    0                    0   \n",
       "1                     0                    0                    0   \n",
       "2                     0                    0                    0   \n",
       "3                     0                    0                    0   \n",
       "4                     0                    0                    0   \n",
       "\n",
       "   batch_UKBiLEVEAX_b5  batch_UKBiLEVEAX_b6  batch_UKBiLEVEAX_b7  \\\n",
       "0                    0                    0                    0   \n",
       "1                    0                    0                    0   \n",
       "2                    0                    0                    0   \n",
       "3                    0                    0                    0   \n",
       "4                    0                    1                    0   \n",
       "\n",
       "   batch_UKBiLEVEAX_b8  batch_UKBiLEVEAX_b9  \n",
       "0                    0                    0  \n",
       "1                    0                    0  \n",
       "2                    0                    0  \n",
       "3                    0                    0  \n",
       "4                    0                    0  \n",
       "\n",
       "[5 rows x 173 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 333428 records (shuffled).\n"
     ]
    }
   ],
   "source": [
    "# Parses the specified phenotypes. It is recommended to look at all the documentation, and in particular at all the \n",
    "# optional arguments, of the create_dataset function. In particular you should double-check that you are really\n",
    "# interested in the behavior dictated by the default values of the arguments (especially those flags which determine\n",
    "# the filtration steps).\n",
    "eid, fields, covariates = create_dataset(PHENOTYPES, nrows = None)\n",
    "\n",
    "# As a result, we get back three dataframes/series, all with the same index (so you can think of them as three pieces of\n",
    "# the same table). Each row in the table represents one sample/individual.\n",
    "\n",
    "# eid is the UKBB phenotype ID of each sample in our project (which appears in the project-specific files).\n",
    "summarize(eid)\n",
    "\n",
    "# This dataframe contains all the fields that have been explicitly requested (described by the PHENOTYPES list).\n",
    "summarize(fields)\n",
    "\n",
    "# The covariates are variables that are always parsed and returned (unless stated otherwise in the parameters),\n",
    "# and should be accounted for if the fields are used as part of a genetic association study. These include:\n",
    "# - const: A constant 1, used as an intercept for regression/classification models.\n",
    "# - sex: 1 for males, 0 for females\n",
    "# - year_of_birth\n",
    "# - PC1-PC40: 40 principle components of PCA over the ~800K genetic markers.\n",
    "# - assessment centers (ACs) in which the samples have been recruited and processed.\n",
    "# - batches: One-hot-encoding of the batch in which the samples have been sequenced.\n",
    "summarize(covariates)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019_12_29-13:59:24] Reading all dataset rows of 345 columns (for 14 fields)...\n",
      "[2019_12_29-14:04:58] Finished after 0:05:34.672725.\n",
      "[2019_12_29-14:04:58] Read a dataset of 502520 samples.\n",
      "[2019_12_29-14:05:00] Knowing of 85 samples who have wished to withdraw, 0 of them are in the loaded dataset. Filtering out these records, the dataset has reduced from 502520 to 502520 samples.\n",
      "[2019_12_29-14:05:00] Of 502520 samples, 472638 are self-reported whites and 409624 are Caucasians according to their genetics. Keeping only the 409592 samples that are both.\n",
      "[2019_12_29-14:05:02] Parsing field sex...\n",
      "[2019_12_29-14:05:02] Parsing field year_of_birth...\n",
      "[2019_12_29-14:05:03] To avoid the \"dummy variable trap\", removing the AC_leeds column (38019 matching records).\n",
      "[2019_12_29-14:05:07] Filtering out 0 samples without genotyping metadata.\n",
      "[2019_12_29-14:05:07] Filtering out 312 samples with mismatching genetic and self-reported sex.\n",
      "[2019_12_29-14:05:09] To avoid the \"dummy variable trap\", removing the batch_UKBiLEVEAX_b2 column (4110 matching records).\n",
      "[2019_12_29-14:05:16] Constructed 60173 kinship groups (147731 samples), of which 52426 (128278 samples) are relevant for the dataset (i.e. containing at least 2 samples in the dataset). Picking only one representative of each group and removing the 75852 other samples in those groups has reduced the dataset from 409280 to 333428 samples.\n",
      "[2019_12_29-14:05:18] Parsing field height...\n",
      "[2019_12_29-14:05:18] Parsing field diastolic_blood_preasure...\n",
      "[2019_12_29-14:05:18] Parsing field red_blood_cell_count...\n",
      "[2019_12_29-14:05:26] Filtering out 58595 samples wihtout any ICD-10 codes, keeping 274833 of 333428 samples.\n",
      "[2019_12_29-14:05:28] Parsing the read dataset into an ICD-10 tree...\n",
      "[2019_12_29-14:09:42] Keeping only 12979 ICD-10 nodes that contain any samples (of 19154 nodes).\n",
      "[2019_12_29-14:09:42] Finished after 0:04:13.988286.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    3751310\n",
       "1    1083290\n",
       "2    3728643\n",
       "3    3467704\n",
       "4    1261121\n",
       "Name: eid, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 274833 records (shuffled).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>coding</th>\n",
       "      <th>meaning</th>\n",
       "      <th>parent_id</th>\n",
       "      <th>selectable</th>\n",
       "      <th>children_ids</th>\n",
       "      <th>samples</th>\n",
       "      <th>n_samples</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>node_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>286</th>\n",
       "      <td>A00</td>\n",
       "      <td>A00 Cholera</td>\n",
       "      <td>23</td>\n",
       "      <td>N</td>\n",
       "      <td>{288, 289, 287}</td>\n",
       "      <td>{130170, 103238, 40322}</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>289</th>\n",
       "      <td>A009</td>\n",
       "      <td>A00.9 Cholera, unspecified</td>\n",
       "      <td>286</td>\n",
       "      <td>Y</td>\n",
       "      <td>{}</td>\n",
       "      <td>{188528, 85795, 126565}</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>290</th>\n",
       "      <td>A01</td>\n",
       "      <td>A01 Typhoid and paratyphoid fevers</td>\n",
       "      <td>23</td>\n",
       "      <td>N</td>\n",
       "      <td>{291, 292, 293, 294, 295}</td>\n",
       "      <td>{130657, 30083, 166828, 197293, 100830}</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>A010</td>\n",
       "      <td>A01.0 Typhoid fever</td>\n",
       "      <td>290</td>\n",
       "      <td>Y</td>\n",
       "      <td>{}</td>\n",
       "      <td>{131705, 23898, 10075, 193767}</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>295</th>\n",
       "      <td>A014</td>\n",
       "      <td>A01.4 Paratyphoid fever, unspecified</td>\n",
       "      <td>290</td>\n",
       "      <td>Y</td>\n",
       "      <td>{}</td>\n",
       "      <td>{128311}</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        coding                               meaning  parent_id selectable  \\\n",
       "node_id                                                                      \n",
       "286        A00                           A00 Cholera         23          N   \n",
       "289       A009            A00.9 Cholera, unspecified        286          Y   \n",
       "290        A01    A01 Typhoid and paratyphoid fevers         23          N   \n",
       "291       A010                   A01.0 Typhoid fever        290          Y   \n",
       "295       A014  A01.4 Paratyphoid fever, unspecified        290          Y   \n",
       "\n",
       "                      children_ids                                  samples  \\\n",
       "node_id                                                                       \n",
       "286                {288, 289, 287}                  {130170, 103238, 40322}   \n",
       "289                             {}                  {188528, 85795, 126565}   \n",
       "290      {291, 292, 293, 294, 295}  {130657, 30083, 166828, 197293, 100830}   \n",
       "291                             {}           {131705, 23898, 10075, 193767}   \n",
       "295                             {}                                 {128311}   \n",
       "\n",
       "         n_samples  \n",
       "node_id             \n",
       "286              3  \n",
       "289              3  \n",
       "290              5  \n",
       "291              4  \n",
       "295              1  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 12979 records (replaced the samples with random indices).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>height</th>\n",
       "      <th>diastolic_blood_preasure</th>\n",
       "      <th>red_blood_cell_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>181.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>4.210</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>158.0</td>\n",
       "      <td>89.0</td>\n",
       "      <td>4.220</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>178.0</td>\n",
       "      <td>83.0</td>\n",
       "      <td>4.800</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>187.0</td>\n",
       "      <td>85.0</td>\n",
       "      <td>4.290</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>162.0</td>\n",
       "      <td>78.0</td>\n",
       "      <td>5.154</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   height  diastolic_blood_preasure  red_blood_cell_count\n",
       "0   181.0                      83.0                 4.210\n",
       "1   158.0                      89.0                 4.220\n",
       "2   178.0                      83.0                 4.800\n",
       "3   187.0                      85.0                 4.290\n",
       "4   162.0                      78.0                 5.154"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 274833 records (shuffled).\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>const</th>\n",
       "      <th>sex</th>\n",
       "      <th>year_of_birth</th>\n",
       "      <th>PC1</th>\n",
       "      <th>PC2</th>\n",
       "      <th>PC3</th>\n",
       "      <th>PC4</th>\n",
       "      <th>PC5</th>\n",
       "      <th>PC6</th>\n",
       "      <th>PC7</th>\n",
       "      <th>...</th>\n",
       "      <th>batch_UKBiLEVEAX_b1</th>\n",
       "      <th>batch_UKBiLEVEAX_b10</th>\n",
       "      <th>batch_UKBiLEVEAX_b11</th>\n",
       "      <th>batch_UKBiLEVEAX_b3</th>\n",
       "      <th>batch_UKBiLEVEAX_b4</th>\n",
       "      <th>batch_UKBiLEVEAX_b5</th>\n",
       "      <th>batch_UKBiLEVEAX_b6</th>\n",
       "      <th>batch_UKBiLEVEAX_b7</th>\n",
       "      <th>batch_UKBiLEVEAX_b8</th>\n",
       "      <th>batch_UKBiLEVEAX_b9</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1944.0</td>\n",
       "      <td>-11.8374</td>\n",
       "      <td>4.223040</td>\n",
       "      <td>-1.755480</td>\n",
       "      <td>-0.217064</td>\n",
       "      <td>-6.907380</td>\n",
       "      <td>-0.046985</td>\n",
       "      <td>0.114862</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1941.0</td>\n",
       "      <td>-11.9716</td>\n",
       "      <td>4.064060</td>\n",
       "      <td>-0.970218</td>\n",
       "      <td>-0.996972</td>\n",
       "      <td>-5.511680</td>\n",
       "      <td>-3.003780</td>\n",
       "      <td>2.267680</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1949.0</td>\n",
       "      <td>-12.4068</td>\n",
       "      <td>7.474360</td>\n",
       "      <td>-2.814510</td>\n",
       "      <td>-1.327240</td>\n",
       "      <td>0.184723</td>\n",
       "      <td>-1.527140</td>\n",
       "      <td>-3.697480</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1943.0</td>\n",
       "      <td>-12.8642</td>\n",
       "      <td>0.906217</td>\n",
       "      <td>-2.587830</td>\n",
       "      <td>3.461230</td>\n",
       "      <td>9.881120</td>\n",
       "      <td>-0.873284</td>\n",
       "      <td>0.330495</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1956.0</td>\n",
       "      <td>-15.3727</td>\n",
       "      <td>5.727580</td>\n",
       "      <td>-0.421423</td>\n",
       "      <td>5.761680</td>\n",
       "      <td>-6.773830</td>\n",
       "      <td>1.922150</td>\n",
       "      <td>-1.410020</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 173 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   const  sex  year_of_birth      PC1       PC2       PC3       PC4       PC5  \\\n",
       "0    1.0  0.0         1944.0 -11.8374  4.223040 -1.755480 -0.217064 -6.907380   \n",
       "1    1.0  0.0         1941.0 -11.9716  4.064060 -0.970218 -0.996972 -5.511680   \n",
       "2    1.0  0.0         1949.0 -12.4068  7.474360 -2.814510 -1.327240  0.184723   \n",
       "3    1.0  0.0         1943.0 -12.8642  0.906217 -2.587830  3.461230  9.881120   \n",
       "4    1.0  0.0         1956.0 -15.3727  5.727580 -0.421423  5.761680 -6.773830   \n",
       "\n",
       "        PC6       PC7  ...  batch_UKBiLEVEAX_b1  batch_UKBiLEVEAX_b10  \\\n",
       "0 -0.046985  0.114862  ...                    0                     0   \n",
       "1 -3.003780  2.267680  ...                    0                     0   \n",
       "2 -1.527140 -3.697480  ...                    0                     0   \n",
       "3 -0.873284  0.330495  ...                    0                     0   \n",
       "4  1.922150 -1.410020  ...                    0                     0   \n",
       "\n",
       "   batch_UKBiLEVEAX_b11  batch_UKBiLEVEAX_b3  batch_UKBiLEVEAX_b4  \\\n",
       "0                     0                    0                    0   \n",
       "1                     0                    0                    0   \n",
       "2                     0                    0                    0   \n",
       "3                     0                    0                    0   \n",
       "4                     0                    0                    0   \n",
       "\n",
       "   batch_UKBiLEVEAX_b5  batch_UKBiLEVEAX_b6  batch_UKBiLEVEAX_b7  \\\n",
       "0                    0                    0                    0   \n",
       "1                    0                    0                    0   \n",
       "2                    0                    0                    0   \n",
       "3                    0                    0                    0   \n",
       "4                    0                    0                    0   \n",
       "\n",
       "   batch_UKBiLEVEAX_b8  batch_UKBiLEVEAX_b9  \n",
       "0                    0                    0  \n",
       "1                    0                    0  \n",
       "2                    0                    0  \n",
       "3                    0                    0  \n",
       "4                    0                    0  \n",
       "\n",
       "[5 rows x 173 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 274833 records (shuffled).\n"
     ]
    }
   ],
   "source": [
    "# The function create_ICD10_dataset is similar to create_dataset (with mostly overlapping arguments). The main difference\n",
    "# between the two functions is that create_ICD10_dataset also returns an ICD10_tree for the samples. The ICD10_tree is\n",
    "# yet another dataframe, with each row representing a node of the ICD-10 tree. The 'samples' column provides the set of\n",
    "# all sample indices associated with each ICD-10 code (these indices are simply the index of the other three dataframes). \n",
    "eid, ICD10_tree, additional_fields, covariates, _ = create_ICD10_dataset(PHENOTYPES, nrows = None)\n",
    "summarize(eid)\n",
    "summarize(ICD10_tree)\n",
    "summarize(additional_fields)\n",
    "summarize(covariates)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Examples working with the phenotypic data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.collections.PathCollection at 0x7fa5920ed0f0>"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAWoAAAD4CAYAAADFAawfAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAgAElEQVR4nO2dfZBcZ3Xmn9M9balHu6jH8YTgtoSEk5JqZWMNngVllaIikdhggZiyTQkv3t1sqNU/WxvsZUWNideWqGQ9KaXW9lbYbGkJAWJHCGR7YiMHzCKn2KiQYOSRkAX2grEteYBYrD3OomlLPT1n/+i+ozvd9+O9t+9n9/OrUmm6+/btc/t2P33uec+HqCoIIYRkl0LaBhBCCPGGQk0IIRmHQk0IIRmHQk0IIRmHQk0IIRlnII6dXnHFFbpmzZo4dk0IIT3J8ePHf6Gqw06PxSLUa9aswdTUVBy7JoSQnkREXnZ7jKEPQgjJOBRqQgjJOBRqQgjJOEZCLSJ3ishpEXlWRPaLyPK4DSOEENLEV6hFpArgDwCMquo1AIoAPhq3YYQQQpqYhj4GAJRFZADAIICfxmcSIYQQO77peao6IyJ/CuAMgBqAp1T1qfbtRGQngJ0AsHr16qjtJCQxJqdnsPcbz+OnszVcWSlj143rMDZSTdss0seYhD6GAHwYwFoAVwJYISK3t2+nqvtUdVRVR4eHHXO2Cck8k9MzuOvRU5iZrUEBzMzWcNejpzA5PZO2aaSPMQl9/A6AF1X1nKrWATwK4F/EaxYh6bD3G8+jVm8sua9Wb2DvN55PySJCzIT6DIBNIjIoIgLgfQB+GK9ZhKTDT2drge4nJAlMYtTHROQggGcAzAOYBrAvbsMI6ZYwseYrK2XMOIjylZVyXGYS4otR1oeq3quq61X1GlX9V6p6IW7DCOmGsLHmXTeuQ7lUXHJfuVTErhvXxWgtId6wMpH0JGFjzWMjVdx387WoVsoQANVKGffdfC2zPkiqxNI9j5C06SbWPDZSpTCTTEGPmvQkbjFlxppJHqFQk56EsWbSSzD0QXoSK3TBCkPSC1CoSc/CWDPpFSjUJHXy3lsj7/aT7EOhJqli5TtbqXRWvjOAXIhd3u0n+YCLiSRV8t5bI+/2k3xAj5qkSt57a+TdfhMY2kkfetQkVfKe75x3+/1g29dsQKEmqZL3fOe82+8HQzvZgKEPkip5z3fOu/1+9ENoJw9QqEnq+OU7Zz1G2sv52mz7mg0Y+iCZhjHSdOn10E5eoFCTTMMYabqw7Ws28A19iMg6AAdsd70DwD2q+kBsVpHEyWp4gTHS9Onl0E5eMBnF9TyAjQAgIkUAMwAei9kukiBZrq5jjJSQ4KGP9wF4QVVfjsMYkg5ZDi8wRkpI8KyPjwLY7/SAiOwEsBMAVq9e3aVZJEmyHF7o9fQ3QkwwFmoRuQzAdgB3OT2uqvvQmk4+OjqqkVhHEiHr4QXGSEm/E8Sj/gCAZ1T1H+IyhqTDrhvXLYlRA70RXsjiAmkWbSLZJ4hQ3waXsAfJN70YXsjiAmkWbSL5QFT9oxQisgLAGQDvUNU3/LYfHR3VqampCMwjeSQLXuPmicOO4ZxqpYwj41sTtcUiizaR7CAix1V11OkxI49aVc8D+JVIrSI9SVa8xiwukGbRJpIPWJlIIiUrqX5ptB+dnJ7B5onDWDt+CJsnDneUufd6S1QSH2zKRIwwDWeYeo1++7M/XhksQRV4o1bHynIJIsDsXN3TjqQXSE2uJHp10ZbED4WaAPAWziDhDJNUP7/9tT/++lx98bmztUt/e9mR9AKp15WE9Zq9uGhLkoFCTXyF00SELEy8Rr/9OT3uhpsdlu2mItjtAqjXlUQWFleDkkebexkKNfEVziCLYCZeo9/+gi6u2T14E4Fp32bL+mE8cnymqwVQtyuJymAp8sXVuEU0KwvC5BJcTCS+whn1Ipjf/oLutyiCyekZbNzzFO44cMKzd7VTf+uHj55x/KG648AJx0XBdianZzB3cb7j/nKpCFVEuriaRH/urCwIk0tQqImvcJo2RpqcnsHIZ/zF0ml/0tp288RhbFk/3PG4Fw1V3HngxJL4tUW7wDiJkFclgZsQWhkea8YP4c4DJ5bE0QGgUi7hvpuvxRsONln79fsRcMoiSUJEmUaYPSjUxFeITZrHW55eu2ABnUJi3x/QFGlLLGdma3jk+Axuub66+LgJXmJrF5gwYtNuv92rdXvtFcsGMDZSxcpyyXW/Xt6wm+fsFF4B3I/LL2XQCaYRZg8KNTES4rGRKo6Mb8WLE9twZHxrR6zSbwGwXUis/VUr5Q6hq9UbePq5czgyvhXS7cFhqcCEFRu7QJosdloes5OXb8fNG3bznIvi/I44HVfYMAlby2YPLiYSAN13qPPzVN0E0iQ+7uZFmmAPqey6cZ1jVooJdoE08cqt1zXBaX9ur9FQRblUNMrFDpKtY8c0jZCZIclBoSaR4CeodrG0f5ndnreyXFrsjWEPjQTFHlLZdfAk9t56HW65vor9x86iYdDnxsK+rd+xBrXXOla74Hm9hkAhAqg2f0Buud75R7abWLPJZHhmhiQHQx8EQLhYph2ny+V23BYWS4Wll/MFAOcvzi+JAUcRAqk3FJ9+9Ps48L1gIg1gSbx8y/ph1+2KIoFEulSQxWO1whN3thZj3ZirL8Ayv6GKR47POJ6vbmLNfp8HZoYkCz1qEol3ZG33ya+c9BRB+5d57zeedxSkBQALjaX7iGoSxVx9IfBzrNCCdanvJaJBfgCqlTLmLs53LMAGPdZuio+cMPk8MDMkWehRk8i8o7GRKhYMhMrEa8wKQ4MlLBso4I4DJyK12WptOuuQJRMGJ7tMFomdMPk8MDMkWehRk0i9I9PFv6zPaiuXirjl+uqSisWobLZ7tZXBkmNKYxic1gBMYs3tC4Imnwc2mEoWCjWJbGbi5PQMXj9/ISqzUqUgwENHz0S+32pbdkTAULknQUNWbiGOleWSY1qh/fPQSw2m8pC9YiTUIlIB8DkA16DpWPy+qn4nTsNIckThHU1Oz2DXwZOoN7LuK5tx/mKw9D0TiiIdk1zcKhfD0h6v9hIhtxDH8lLBKAWwF4YOt39urewgIFvZK6Ye9YMAvq6qt7amkQ/GaBNJmCi8oz1PnO4ZkY6LhipGPvPUYm/tyqB71WI3WCEKv0VBtxDH7Fwd9+/YmHkvMwqcPrf1hmLPE6czdby+Qi0iKwG8F8DvAYCqXgRwMV6zSNKE9Y4sjy2qOGuvY3+f4nrPyqUCrr7rSccMFLvH7RbyUjSzd257zyr80di1oe3IQ0jB7Rxk7fNs4lGvBXAOwF+KyHUAjgP4RGuO4iIishPATgBYvXp11HYSD9L6QrR7bCQb+KUgWj2yz1/o7Phn0VBdjNHbxdr0s8aCmGjxnUIuIqMAjgLYrKrHRORBAP+oqv/Z7TmcQp4cTmJZLhWN0rC6xW2qNsk2lXIJ5y/Mo77gH6oqiuCF+24C0IrnfvXkkueVCoId716Fp587t0S83fLNszZxfeOepxwXTivlEk7ce0OitnhNITfJo34FwCuqeqx1+yCAd0VlHOmONCrErKo1inQ+mbtoJtJA07O++q4ncffkKex+/HTH8+oLTc/bXlm56+DJwF3+0mL39g0dlbGlgmD39g0pWeSMb+hDVX8uImdFZJ2qPg/gfQB+EL9pxISkKsTsVXnd9N4g6XMx4KKvPQxigteictYKYvKSZmia9fEfADzcyvj4CYB/G59JJAimw2S7+SC2h1co0iQM7Z0MAWeBTHrNJQ9phr4x6jAwRp0cfjHqKGLYDHOQbmm/CisVBJCl3rdTNah1fxJrLmnjFaOmUGeIsJ6E/XmVwdJinu6VlTLOX5h3XCxpX9T52P/8Do688Nri7c1XX46H/91vAgDWjh+iF00SoSjimFZYtS1SOn3Ovb4reUgTBCjUuSAKzzdoupz14f/q1JklIm2nXCqgFqLjHOld0lqjaK+WtFMqCvbeep3jcIOw36ukBZ5CnWH8WmcGSWcKE6Lw+vATkjRuHrXb/XaGBkuYvmdpSp3bd8Lve5VG2mu36XkkJtqHpDoRJHsj7OBWQrJAuVTEbe9Z5Tih3qTPt1M1odt3wm8KfNYGI1CoU8RkSGqQdKaspT4RYsqygQKWDRTw8NEzWF4qoNKa3t5tmMXrO+E17DdrgxEo1Cnid9JNO9iZFqC4DLAmJHUuzC9gtlaHoukZX5hfwNBgKZBIi6BDdP1GxNXqDXzyKyc7npe1wQgU6hRZWXbvnmY6jcMkfGIRw3IEIbFQqzcCN0ZSBXYdXCq69ik3bjRUHWd5OoVg/EImcUGhThE3D3dosIQj41uNFi1MwieE9AtWi1I7YyNVHBnf6inW7fHndoG3h2C8QiZxwQkvKeI2L8/tfnu60MpyCSLZa8dISNq0fydM2x+0hyKtikWnsKLbQOG4oEcdA1bMeO34Ic/LJLd4lwIdz7M6l1nNb2ZrdYo0IT60hwa9on9u38csLCzSo44Ytz68Uy+/1tEK0mkElkX7SCCnzmWE9CtDHkOBK+WSb31Cu2fttXAf1UzRbmDBS8S4ZV84fTDuu7nZkN3rA7XisiIqg5ex1wYhNkoF8XRcTNL6qpWyUdWhW/HLLddXO5yvbkIhXgUv9Kgjwu8XvP1DU6s3cMeBE4tl3HccOOH4vPMXGzh/kSJNiB2/q0sTkTat+HVqhbpl/fCS5lFxT7ChR90FUfVo9vMOCCHRYX1Xq114wWFL0z3tokcdPVH2aKZIE5IMTml2wCUv+O7JU9h/7CwaqiiKuA74TXqBkVkfIWH+MiH5wykEaeVP3z15Cg8dPbPYV8SabHP35KmO/SRduWgk1CLykoicEpETItL7MQ0Dsjb7jRASDiuEsf/YWcfHne53qlw0bfkQhiAe9RZV3egWQ+k32ACJkGxQrZQxNOjejsGEuydPuXboa6h21EYAWKxcFJi3fAiL0WKiiLwEYFRVf2Gy035YTAzapJ8QEj0FAX5y37ZmQdjBk56Ddbt5jWUDxdh7U0fRj1oBPCUix0Vkp8uL7BSRKRGZOnfuXFhbc4NJsxdCSLwsKLBm/BA++/SP0PAR6ULI7pHLBgqp96Y2FerfUtV3AfgAgH8vIu9t30BV96nqqKqODg8PR2pkVrGavRBC0uVHr56H18C4cqmIZQPBcycKgOsouiTXqYwsV9WZ1v+vAngMwLvjNCpvFNnomZDMUhTBLddX8WaI2Z9ez0hynco3j1pEVgAoqOr/a/19A4DPxG5ZSgQZaGltazImiBCSDg1VHPjuWVQ8+oMEJc4MDydMCl7eCuAxaXqNAwD+WlW/HqtVKeHWUAnoLAvlYiIh+aG+oHiz3ohkmHM3FY1h8Q19qOpPVPW61r8NqvrHSRiWBkEGWrLghZB8Uasv4JbroxHXOw+cSHTSC0vIbQQpC2XBCyH548B3nYtagmAVyMTdiMkOS8htBCkLZcELIfkj6r46SaXpUahtBCkL9ZtuTAjJDqVi8Mws02ckcXXN0IcNe9/ZmdkaiiKo1RvY/fhp7HniNGbn6h2ZIF49qAkh6ePX8x1oToWZrS3NCFGYtSBO4uqaQt2GJcD2jA77CbTHpQgh2ccqStvzxGnH9LwVlxWxYtlAh1ADzVCJV6/5pNL0+nZwgFe+tFtTcDtDgyW8WV9g5gchGcfyqAF09AMpFgQF+MeundL6hgZLuPdDGyJbSIyi10dPYZ9MrLjkJVupNiYxp9fn6hRpQnKAfVD03luvW+zPUxRBY0GNFhjbO+U9sGMjpu+5IbFc6syEPoJUBHaLV7702EjVdeowISSf1BuKPU+cxvQ9NwBAoGK1gjRDol56FLd+ZcKj9vNwo8YvX3rXjes8V4lLRcFgKRNvHSF9T7un68brc3VsnjiMPU+cDnQ17OdwJ6FfmfCo/TzcqHHzmK+slDE5PYM9T5z27mur3c1IJIREw+2bVnfMNPTK7ghzpVwpew8lSEK/MuEWJj0o0i1fesv6Yew6eNK3cUt9QV1bHxJC4qco4ijSceDXHDMJ/cqER+3l4caBPQfaHlPy9aQJIZnghftucn3MKSe6G2Z9HLck9CsTHnXSgyKBS03/X5zYhiPjWzE2Uo2sBSIhJF68GiLt3r4BJZ9xLpVyyXg6k5/gJqFfmRBq+1irJAZFEkLyjdeC3dhIFXs/cikNr12yy6Uidm/fgCPjW33LxE0ENwn96tuCFyc27nnK9ZJpKMKm44SQaKhWyr7j8MIWt1XKJYjAsXVEHOS24KV9RHvcvV/dLpmsCqTbN62O9fUJIcHodsHOLWxx+6bVuDC/gNfn6omkDPthLNQiUhSRaRH5WpwGWSSdWw1cumRqT8d5fa6Oux49ha+d/Flsr00ICY5f/NhPR9zCFk8/dy71yeN2gnjUnwDww7gMaSfItJUoGRupYsWyzmSYWr0R6UoyIaQ7TOLHJjrilFiQdMqwH0ZCLSJXAdgG4HPxmnOJNN8oTm8hJNuYLtiF1ZEgQ0SSwNSjfgDAp+AxPV1EdorIlIhMnTt3rmvDonqjwsS5Ob2FkOxiLSCaLOyF1ZE0Uoa98BVqEfkggFdV9bjXdqq6T1VHVXV0eHi4a8OieKPCxrndXtuvlJQQEi9BNSCsjmQtZdikMnEzgO0ichOA5QDeIiIPqertcRrmVj0Y5I0KW4M/NlLF1MuvYf+xs2iooiiCW66vYvTtlwfqukUIiY5qCA3oRkf8OuYlSaA8ahH5bQD/SVU/6LVdVvKo144fcmyeJABenNjm+jzLE7cLcrlUxH03N/sKcPwWIclhffecRDPJ9shxk9s86m4JG5/y88SPjG/FAzs2crgtITFhVTN4hRzSSOFNi0BNmVT17wD8XSyWxMCuG9c5esZ+8SmTlWIOtyUkHkxDHEm3R06TTHTPi4uw8SnTblhWDGvN+KHojCakjxHAtyTcImu5znHS00INhFsQCOuJE0L8qVbKmLs479g7J0hqbNLtkdOkp2PUYQmamlP06yxOCAFwKQd62zvf5vj4lvXmqb1Zy3WOk573qMMSxBO/7T2r8NDRMx33F+BRIURIn2EX0aefcy6Kc7vfiShSePMChdoAvxQgaxyQPe/6tveswujbL+diIyHoXCCMKr6cpVznOKFQ+9CeU22lAAHoEGun+W1jI1XHvGxCsszQYAmqwBu1Oiqtv92akgnc48Vu9FN8OQoYo/Yhii5+9pg3IXlg8LIB7N6+AS9ObMP0PTfgxL03uG6rgG9tQXuOcz/Fl6OAQu1DlJdoR8a3UqxJJvD7HDoVj7g9x7rfzyGxOzhZ66WRdSjUPkTd7tDJkyAkSaxcZT+xbr9yNPGCLYfELQ+qvWisvQ80cYZC7UNUl2hWu9U7D5zAsoEChgbZiY+kw5WVMianZzA7d9F323ZhNfWCs9bPOe9wMdGHKFKA2hcTZ2t13+nHhASlKIKGT5O1cqmILeuHjRe33apx/WDRWLRQqA3oNgXIaUEy+tnvpJ8pFYAf/ZebPNsZWFO1nXL+nehGWPspxzkJKNQuRNk+sRd7D5BsMW9QWXVhfsE4RTRM7+d2+iXHOQko1A6Y5k6bYppjan057EMLCDHBJPZrKtKVcsm4MRJJBi4mOhD1BHSTTA+rBwIAHPguRbqfCbp+IWg6E5snDnf92qWCYPf2DV3vh0QLPWoHom6f2N67WrA0Rm2PBe5+/DTqC/4ibcUbZ+fqKJcKqM0vgNreG/idxiFbpaD9s+T02bLwWmi0Hosi3EHiwVeoRWQ5gG8DWNba/qCq3hu3YWkSR3mrPV7nFf92K9Ntx14pZu2PPUX6A1XAatjYLr0KODoCt1xfxSPHZxzHywXNYOICYfKYeNQXAGxV1V+KSAnA34vI36rq0ZhtS424U4uiXGRhH5H+w+/HXNEMpbWLqdUkLKo0027Xbog5vkKtzem3v2zdLLX+9fRFdpqpRUODJceG6naKIvSiiStuxVRxpJn26uirrGEUoxaRIoDjAH4dwGdV9ZjDNjsB7ASA1atXR2ljKqSVWnTvhzZg18GTqDfcfws3vWOIXnQOKBYEb1k+4PvD60W5VAx0nosFwS/fvDQ9JUqvt59GX2UNo6wPVW2o6kYAVwF4t4hc47DNPlUdVdXR4WHzKQ1kKWMjVez456tcp8ZsvvpyvPR/axTpHNBYUKjCqLeLdb6HBkvNheLW38sGgiVmNRa0YzG6m4wlOywLT4+gU8hnReRpAO8H8Gw8JvU3k9MzeOT4jOsK/TNn3ggs0isuK+LN+gJT/lLgjVod9+/Y6Jnx47SgF/XaQxReL8vC08Mk62MYQL0l0mUAvwvgT2K3rE9xigPaqdUbnqlW7UJQKgouzlOk0+LKStk448eO1+fASs0MElKJwutlWXh6mHjUbwPwxVacugDgK6r6tXjN6l9MFgcbqh2xS8szAy7laxdFPGPdxJ+gMeL25+66cR3unjzVMabNaRqQHTcPWHApNXPzxGHHz4tXnn63sCw8HXwDYKr6fVUdUdV3quo1qvqZJAzrV0wmmlvtJZ3aTY6NVBcrIYN40UURlFin2oH9fbbixyaIAMtLBdxx4AQeOnpm8Vw0VPHQ0TO4e/KU5/NN4sFuLXg/tmk1G/L3GKxMzBgmbSqty023L59f+MTtdeNyvrvxStPG6X3euOcp/1xm9Q5N7D921tOrNokHMxTRP1CoM0bVo4GTW4lve9wza7nV9918Le44cCJtM0KxeeJwx3tuWj3qhd8PsqkIMxTRH/BiN2O4Xc4+sGOj47giKztgZrYGxaV+D1libKQayqZiIZojKZcKocefOc0OjAKTEJcb1rSgteOHsHnicOS2kewhGkM2wOjoqE5NTUW+334hSD8F0wUl08fiwOsqIQmGBku490Mbuq7ktK5oorg6uH3Tas+S7snpGXzyqyfRsOVEFwuC2969KpKeHWnCfiHOiMhxVR11fIxCnW/Wjh9yFV2r30Ol1W3tjVp98Yux+/HTkVzC5wEB8OLEtsXb9iwMATB4WRHnL3qnPdr31c03xsr6GH375Y4xaEtwN9zzdZy/2BnXd3t9e5vcICQtmk754Xn7oYkLL6FmjDrnuMWkTb64u7560qilat5RLI01/9HYtYsLeZZQzV2s4ddWLsfcxXnPRcBu3i0B8MJ9NwEte7z6ZjiJtNfrhyloSaPJEvuFhINCnXNMsgPcvKYsTJKpVsrYsn7YeI5fWOwiBDj3Bp+ZraFUEJSK8eSf21Prou6bEaagJQ3RZL+QcFCoM47fpalfdoCb1zT18muepep2VrRCA1FjTzV87JmZWF7DTq3ewO7HTy+ZHdh+9PUFRaVcwoplA5HG1dt/PP16nnuFWJyKncIUtKQhmnH0eu8HmPWRYZwyOpwyEMZGqjgyvhUvTmzryAxx85r2HztrnNscpYBauQ5Ww6E7D5zA5onDWEjIq5+t1X2P+41aHUfGt+KBHRtDZ4vYcSo6ccvusQT3Y5ucO1Devmm1a7FTUNJosuR33MQZetQZJopLUzfvKMlwh+UdWlkTADq8/CxhCVX7CLWgeC2S+V0JWTF0t9LzKEITaTRZYpFOOCjUGSaKS1O3S02TDIcoqJRL2L19w5IvotNCWhKYZmycvzCPyemZrsSjKOLr6foVq9gXPeMgLdFkkU5wKNQZJop4npvX5DRDLy7av5RxeND2jnLWj5DbEFg/Zmv1JXF8t/coylmEaUHRzAcU6gwTxaWpl9dkFVzEGXqYrdWxeeLwktd28+YLArxtZbjimBXLBly9QbeiIC+sOL7bVUfV4X3kpTyJCxa8ZJygBQlhChhMqxut205tNE09c79tX5rYhjXjh4z25bRvJ0827P7caC+gISQKvApemPWRcbwyOtoxzRJpx7Rd5v07NuKliW24f8fGjqwD09YVXiJt9b+ouoR2KuUSSh79P2r1Bu5oZZGY9r8I03ODqWQkaRj66CHCZokEXVRyimtG0f/CCjO4FcDUGwtGlZRBKuxM2spy9BRJG5NRXKsAfAnAW9G84t2nqg/GbRgJTjdZImEXlaxQSxRYnvTTz51zfDxIPncUFXZWHJrxZ5I2Jh71PIBPquozIvJPARwXkW+q6g9ito0EJOmqL78BrKWCAAKjcmy7pxpVZZy1n3KpgFp9IdBzTQY0EJIUgRcTReRvAPyZqn7TbRsuJqZD0M5k3XZO88qmsBe3+GWWDLV19zt/YT6Szn5WY6rJ6Rn8xwMn4CfVVrfBla1Uv9m5umPnQQo3iYPIuueJyBoAIwCOOTy2E8BOAFi92rn8lcRLkFhzFJ3TvAaw2jv3jY1UPTMvfvnm/GLseWa2hoI0vfFuOvvZPXSTCsNKubQo6vb3xd5JL4nucoQ4YSzUIvJPADwC4A5V/cf2x1V1H4B9QNOjjsxCEgjTS/UoytODhFoq5ZKrl9wuyAsKLB8Q/OpblhvnPw+WCqjNL0C1mclxy/VL3wfrfZmcnulo71oqCHZv3wDAf94kW3KSNDBKzxOREpoi/bCqPhqvSSQJull4tEZBOY39csuK2L19Q0dqnVeq3ZwtpuyVQFcqCG7ftBoKgRXFa6jikeMzjil6YyNV7P3IdUvSC/d+5LpF4TU5/pnZGkdgkUQxyfoQAH8B4Ieq+l/jN4kkQdiFx/bQgL0Axm34LuAelvFK67Pss/vbBQGWDRTwZn1hcR9Brw68rjpMhwMzDEKSxHcxUUR+C8D/BnAKWFyP+bSqPun2HC4mZp+wI5HcFhDDjoIa+cxTnhNVnGi302QcmelCoF8mi9P+wxw3Ie10VZmoqn+vqqKq71TVja1/riJN0sV0QvXYSDVUX2O30MDMbA1X3/Uk7p485fi4G9ve+bZA2wOXvGULt6sAadkVpEqz/X0ZGiyhUi65bs/JJCQJWJmYcYKk0AXN5AiTI+wVGmioLlYUmrbndCtu8cMukE7Nq5y65ZkuBDq9L25XEiwnJ0nAXh8ZJmjvDq9YrclrmXjiTn1B2tl/7Kzv61mE7dxnF0inq4Moh8ACnExC0oUedYYJukgWNpMjiCdukpMcZCBBmJanpaJ0CGS7FxyVB2y/ollZLmF5qYDZORa/kGShR51hggpv2Bl4QZnHqnwAAAkLSURBVD1xq6OfW+e5IB3p3ER9QbE4t7Ajjc/gd8DUA/a6kmi/opmt1fFmfQH379jo28mQkCihUGeYoMIb9vI8rCd+23tWBbrfCbeWptb9e7/xfEdBTH1BfcM5JoulfqGlbkJJhEQJQx8ZJuiEl7Az8MLmVPsNYDXB7Ri3rB/27CUSRUdAv9BSFDMrCYkCCnWGCSO8YTI5uhn51e0AVqdj3LJ+2HeeYxTZFn5CnHQ3QkLcoFBnnCTabKY1jdr++kGmlEeVbeEnxFHMrCQkCijUBECwH4Ru26P64RVa8CpTD4qfEKf9A0aIBYWaBCKK9qh+uHm6UZdrmwgxBweQLMAp5CQQUff6cMKrDwlAD5f0JpENDiAkiUwIN08XQOzePCFZhEJNApFUJoRbv41uhx0QkkdY8EIC0U3PC9N+Im4wr5n0K/SoSSDCZkJEsQjJvGbSr1CoSWDCZEJEMaORec2kX/ENfYjI50XkVRF5NgmDSG8SRdgi7LADQvKOiUf9BQB/BuBL8ZpC8kKYgpeowhbMayb9iMkorm8DeC0BW0gOCDrMwIKN9wkJT2RZHyKyU0SmRGTq3Llw45VI9gnb+pNhC0LCE9lioqruA7APaFYmRrVfki26iTUzbEFIOJhHTQIRdooMISQ8FGoSCMaaCUkek/S8/QC+A2CdiLwiIh+P3yySVRhrJiR5fGPUqnpbEoaQ/JD3WHPc/bQJiRpWJpK+ImgpO0WdZAHGqElfESS9MGzOOCFRQ6EmfUWQ9MKwOeOERA2FmvQVQdIL2VaVZAUKNekrgqQXMmecZAUKNekrgqQXMmecZAVmfZC+wzS9MOyQBEKihkJNiAd5zxknvQFDH4QQknHoUZO+gIUrJM9QqEnPE8Vg3bThD01/w9AH6XnyXrjCCklCoSY9T94LV/L+Q0O6h0JNep68F67k/YeGdA+FmuSWyekZbJ44jLXjh7B54rBrKCDvhSt5/6Eh3UOhJrnEKW5754ETWOMg2ibViKainwZ5/6Eh3SOq/nNoReT9AB4EUATwOVWd8Np+dHRUp6amorGQEAc2TxzGjMelf7lUNJ48054VYn8+kI3KRHvWx8pyCSLA7FydGSA9hIgcV9VRx8f8hFpEigD+D4DfBfAKgO8BuE1Vf+D2HAo1iZu144fg52JUK2UcGd/quy830R8aLOHN+oKjgKcljF4/KhTrfOMl1Cahj3cD+LGq/kRVLwL4MoAPR2kgIUExic+aLra5bff6XD1z2RbMAOlPTIS6CuCs7fYrrfuWICI7RWRKRKbOnTsXlX2EOOIUt23HdLEt6KJcmtkWzADpTyJbTFTVfao6qqqjw8PDUe2WEEfsC4QAIG2PB1lsc1usq5RLjtunmW3BDJD+xKSEfAbAKtvtq1r3EZIq9s523ZRYu7UzBeAYD04z22LXjesyZxOJH5PFxAE0FxPfh6ZAfw/Av1TV027P4WIi6RWy2GMjizaR7ukq66O1g5sAPIBmet7nVfWPvbanUBNCSDC8hNqoe56qPgngyUitIoQQYgQrEwkhJONQqAkhJONQqAkhJONQqAkhJOMYZX0E3qnIOQAvR77jeLgCwC/SNiJGev34gN4/Rh5f/jE5xrerqmO1YCxCnSdEZMotJaYX6PXjA3r/GHl8+afbY2TogxBCMg6FmhBCMg6FGtiXtgEx0+vHB/T+MfL48k9Xx9j3MWpCCMk69KgJISTjUKgJISTj9J1Qi0hRRKZF5Gut22tF5JiI/FhEDojIZWnb2A0iUhGRgyLynIj8UER+U0QuF5FvisiPWv8PpW1nWETkThE5LSLPish+EVme93MoIp8XkVdF5FnbfY7nTJr8t9axfl9E3pWe5Wa4HN/e1mf0+yLymIhUbI/d1Tq+50XkxnSsNsfp+GyPfVJEVESuaN0Odf76TqgBfALAD223/wTA/ar66wBeB/DxVKyKjgcBfF1V1wO4Ds1jHQfwLVX9DQDfat3OHSJSBfAHAEZV9Ro02+5+FPk/h18A8P62+9zO2QcA/Ebr304Af56Qjd3wBXQe3zcBXKOq70Sz3/1dACAi/wzNc7qh9Zz/3hqwnWW+gM7jg4isAnADgDO2u0Odv74SahG5CsA2AJ9r3RYAWwEcbG3yRQBj6VjXPSKyEsB7AfwFAKjqRVWdRXMY8Rdbm+X6GNFszVtuDbQYBPAz5Pwcquq3AbzWdrfbOfswgC9pk6MAKiLytmQsDYfT8anqU6o637p5FM3JUUDz+L6sqhdU9UUAP0ZzwHZmcTl/AHA/gE8BsGdshDp/fSXUaA4/+BSAhdbtXwEwa/vAOA7uzRFrAZwD8Jet8M7nRGQFgLeq6s9a2/wcwFtTs7ALVHUGwJ+i6aH8DMAbAI6jt86hhds5Mxo2nTN+H8Dftv7uieMTkQ8DmFHVk20PhTq+vhFqEfkggFdV9XjatsTIAIB3AfhzVR0BcB5tYQ5t5mPmMiezFaf9MJo/SFcCWAGHS85eI8/nzA8R+UMA8wAeTtuWqBCRQQCfBnBPVPvsG6EGsBnAdhF5CcCX0bxcfhDNSw9r0k3eB/e+AuAVVT3Wun0QTeH+B+vyqvX/qynZ1y2/A+BFVT2nqnUAj6J5XnvpHFq4nbOeGTYtIr8H4IMAPqaXCjp64fiuRtOZONnSm6sAPCMiv4aQx9c3Qq2qd6nqVaq6Bs3FisOq+jEATwO4tbXZvwHwNymZ2DWq+nMAZ0XEGkn9PgA/APA4mscG5PsYzwDYJCKDrfUF6/h65hzacDtnjwP4163sgU0A3rCFSHKDiLwfzTDkdlWdsz30OICPisgyEVmL5qLbd9OwMSyqekpVf1VV17T05hUA72p9P8OdP1Xtu38AfhvA11p/vwPND8KPAXwVwLK07evy2DYCmALwfQCTAIbQjMV/C8CPAPwvAJenbWcXx7cHwHMAngXwVwCW5f0cAtiPZsy93vpSf9ztnAEQAJ8F8AKAU2hmwKR+DCGO78doxmpPtP79D9v2f9g6vucBfCBt+8McX9vjLwG4opvzxxJyQgjJOH0T+iCEkLxCoSaEkIxDoSaEkIxDoSaEkIxDoSaEkIxDoSaEkIxDoSaEkIzz/wHzv5yaytxJiAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Example 1: A graph showing diastolic blood preasure vs. red blood cell count\n",
    "fig, ax = plt.subplots()\n",
    "ax.scatter(additional_fields['diastolic_blood_preasure'], additional_fields['red_blood_cell_count'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[Text(0, 0, 'F'), Text(0, 0, 'M')]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8li6FKAAAWnklEQVR4nO3df2zU933H8efbZ4OL2QourGohga6B7sKta1M3ZZunxWk3QtcUIm1tXKmk5RSrWfDSLBWFnbRsf1jqWH6osbUiJlshWriSbW2KumSsC6dFJw0mU7WpG28DraExSxt3JGlwajg77/3hr62zY7B95/PX/tzrIZ3ue+/v98wbOL348rnP9/M1d0dERMJSE3cDIiIy/xTuIiIBUriLiARI4S4iEiCFu4hIgGrjbgBgzZo1vnHjxrjbEBFZUk6fPv0zd1873b5FEe4bN26kt7c37jZERJYUMzt3pX0alhERCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQqLpvNkkqlSCQSpFIpstls3C0Fb1FMhRSRcGWzWTKZDN3d3TQ3N5PP50mn0wC0trbG3F24bDEs+dvU1OSa5y4SplQqRWdnJy0tLRO1XC5He3s7fX19MXa29JnZaXdvmnafwl1EKimRSDA8PExdXd1ErVAoUF9fz+joaIydLX1XC3eNuYtIRSWTSfL5/KRaPp8nmUzG1FF10Ji7iFRUJpPh05/+NA0NDZw7d44NGzYwNDTEV7/61bhbC5rCXUQq7vXXX2dwcBCAF154gfr6+pg7Cp+GZUSkovbs2UOhUODBBx9kaGiIBx98kEKhwJ49e+JuLWj6QlVEKsrM2LRpE2fPnsXdMTOuu+46zpw5w2LIn6Xsal+oalhGRCruzJkzE9vuPum1VIaGZUREAqRwFxEJkMJdRCRAM4a7mV1jZjkze97Mfmhm90T1RjP7jpmdiZ5XR3Uzs0fM7KyZPWdmN1T6NyEiS8Ojjz4adwtVYzZn7iPAfe5+PbAVuNvMrgf2Ac+4+ybgmeg1wHZgU/RoA742712LyJL0uc99Lu4WqsaM4e7uL7n7d6Pt14F+YB2wAzgcHXYY2Blt7wAe8zEngVVm9q5571xERK5oTmPuZrYR+CBwCninu78U7foJ8M5oex3wYtHbBqKaiIgskFmHu5mtBP4R+KK7/7x4n49diTCnqxHMrM3Mes2sd/yyZBERmR+zCnczq2Ms2B93929E5Z+OD7dEzy9H9fPANUVvXx/VJnH3Q+7e5O5Na9euLbV/ERGZxmxmyxjQDfS7+0NFu44Bd0TbdwDfKqrvimbNbAVeKxq+ERGRBTCb5Qd+G/gs8AMz+15U+zPgK8ATZpYGzgGfivY9BXwcOAu8AXx+XjsWEZEZzWa2TN7dzd3f7+4fiB5Pufv/uftH3X2Tu3/M3S9Ex7u73+3u73X3X3d3rQi2QHQTYhEZpytUA5HNZrnnnnsYGhoCYGhoiHvuuUcBL1KlFO6B2Lt3L7W1tfT09DA8PExPTw+1tbXs3bs37tZEJAYK90AMDAxw+PBhWlpaqKuro6WlhcOHDzMwMBB3ayISA4W7iEiAFO6BWL9+Pbt27SKXy1EoFMjlcuzatYv169fH3ZpUITObeMz2uJmOlblRuAfiwIEDDA0NsW3bNpYtW8a2bdsYGhriwIEDcbcmVcjdJx6zPU633JtfCveATD3z0ZmQLAZXuhG2bpBdWQr3QOzdu5cVK1Zw/PhxLl++zPHjx1mxYoVmy0jsOjs72bNnD8uXLwdg+fLl7Nmzh87Ozpg7C5sthv8KNTU1eW+vrnUqh5nxyU9+kuPHj3Pp0iWWL1/Otm3bOHbsmP67K4uGmenzOI/M7LS7N027bzH8QSvcy3e1IZjF8HcsAgr3+Xa1cNewTGBWrlw56VlEqpPCPTAXL16c9Cwi1UnhLiISIIW7iEiAFO4iIgFSuIuIBEjhLiISIIW7iEiAFO4iIgFSuIuIBEjhLiISoBnD3cx6zOxlM+srqn3AzE6a2ffMrNfMbozqZmaPmNlZM3vOzG6oZPMiIjK92Zy5PwrcMqV2APhLd/8A8OfRa4DtwKbo0QZ8bX7aFBGRuZgx3N39WeDC1DLwy9H224H/jbZ3AI/5mJPAKjN713w1K5OVciszEakOtSW+74vAcTN7gLF/IH4rqq8DXiw6biCqvTT1B5hZG2Nn91x77bUltlHdipdO1ZK/EofGxkZeeeWVOb1nricZq1ev5sKFqeeXMpNSv1C9C7jX3a8B7gW65/oD3P2Quze5e9PatWtLbENE4vTKK6+85T6o8/2Y6z8eMqbUcL8D+Ea0/ffAjdH2eeCaouPWRzWpsCudneusXaQ6lRru/wv8brR9M3Am2j4G7IpmzWwFXnP3twzJSGUU30Fed5MXqW4zjrmbWRa4CVhjZgPA/cCdwFfNrBYYJho7B54CPg6cBd4APl+BnkVEZAYzhru7t15h14emOdaBu8ttSkREylPqbBkREfz+X4a/eHvlfw2ZM4W7iJTM/vLnFf9ux8zwv6joLxEkrS0jIhIghbuISIAU7iIiAVK4i4gESOEuIhIgzZYRkbJUerXR1atXV/Tnh0rhLiIlm+s0SDPTshgLRMMyIiIBUriLiARI4S4iEiCFu4hIgBTui1xjY+Oke6DO5gHM6fjGxsaYf5ciMt80W2aRG7+NWSXpxtki4dGZu4hIgBTuIiIBUriLiARI4S4iEiCFu4hIgGacLWNmPcAngJfdPVVUb2fsZtijwD+5+96ovh9IR/U/cffjlWi8WugelSJSitlMhXwU6AIeGy+YWQuwA/gNd79kZr8S1a8Hbge2AO8G/tXMNrv76Hw3Xi10j0oRKcWMwzLu/ixwYUr5LuAr7n4pOublqL4D+Lq7X3L3HwFngRvnsV8REZmFUsfcNwO/Y2anzOzfzOzDUX0d8GLRcQNRTUREFlCpV6jWAo3AVuDDwBNm9qtz+QFm1ga0AVx77bUltiEiItMp9cx9APiGj/kP4E1gDXAeuKbouPVR7S3c/ZC7N7l709q1a0tsQ0SWgunWPZLKKjXcnwRaAMxsM7AM+BlwDLjdzJab2XuATcB/zEej1WyuC4fN9aHbmEklXSnIFfCVNZupkFngJmCNmQ0A9wM9QI+Z9QGXgTt8bErHD83sCeB5YAS4WzNlylPKTBndykziNtvgnnqcPrfzxxbDH2ZTU5P39vbG3UYwFO6ymFwt6PU5LY+ZnXb3pun26QpVEZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQWzL333ht3C1VD4S4iC+bhhx+Ou4WqoXAXEQmQwl1EJEAKdxGRACncRUQCpHAXkQVRX1/PyZMnqa+vj7uVqlDqbfZEROZkeHiYrVu3xt1G1VC4B6R43ezxba2XLVKdNCwTCN3KTBa7lStXTnqWytKZ+xJWyq3MdCYvcTAzLl68CMDFixd1t7AFoDP3JczdJx7zcZxIpbg7iUQCgEQioc/iAlC4B+jLX/5y3C2IvEVbWxuvvvoqbW1tcbdSFXSD7EDoJsSyWJkZ9fX1jI6OUigUqKurI5FIMDw8rM9mmcq6QbaZ9ZjZy2bWN82++8zMzWxN9NrM7BEzO2tmz5nZDeW3LyJL3W233cbmzZupqalh8+bN3HbbbXG3FLzZDMs8CtwytWhm1wC/D/y4qLwd2BQ92oCvld+iiCxljY2NHD16lN27d/P666+ze/dujh49SmNjY9ytBW3GcHf3Z4EL0+x6GNgLFP+/agfwmI85Cawys3fNS6cisiR1dXWxcuVK9u3bR0NDA/v27WPlypV0dXXF3VrQSvpC1cx2AOfd/ftTdq0DXix6PRDVpvsZbWbWa2a9g4ODpbQhIktAa2srBw8enDQsc/DgQVpbW+NuLWhznuduZiuAP2NsSKZk7n4IOARjX6iW87NEZHFrbW1VmC+wUi5iei/wHuD70QyN9cB3zexG4DxwTdGx66OaiIgsoDkPy7j7D9z9V9x9o7tvZGzo5QZ3/wlwDNgVzZrZCrzm7i/Nb8siIjKT2UyFzAL/DrzPzAbMLH2Vw58C/gc4C/wt8Mfz0qWIiMzJjMMy7n7VgbLo7H1824G7y29LRETKoeUHREQCpHAXkYrLZrOkUikSiQSpVIpsNht3S8HTkr+Bqaurm1i/o1AoxN2OCNlslkwmQ3d3N83NzeTzedLpsa/uND2ycrRwWCBqa2sZHR19Sz2RSDAyMhJDRyJjUqkUO3fu5Mknn6S/v59kMjnxuq/vLUtWyRxcbeEwnbkHYnR0FDOjpqaG0dFREokEb7755rSBL7KQnn/+eYaGhujp6Zk4c9+9ezfnzp2Lu7Wgacw9ILfeeisjIyO4OyMjI9x6661xtyTCsmXLaG9vp6Wlhbq6OlpaWmhvb2fZsmVxtxY0nbkH5NixY9TV1TEyMkJtba2GY2RRuHz5Ml1dXXzwgx+cOHPv6uri8uXLcbcWNIV7IBKJBKOjoxM3Pxh/Hr+1mUhcrr/+enbu3El7e/vEmPtnPvMZnnzyybhbC5qGZQLh7tTU1HDgwAGGhoY4cOAANTU1utONxC6TyXDkyBE6OzsZHh6ms7OTI0eOkMlk4m4taJotEwgz4+abbyaXy+HumBktLS2cOHFCAS+xy2azdHR0TJy5ZzIZTYOcB2XdZk+WhtraWk6dOsWGDRuoqalhw4YNnDp1itpajbyJVCOFeyCWL1/O0NAQ27dv58KFC2zfvp2hoSGWL18ed2tS5cYvYioelslkMrpKtcI0LBMIM2Pjxo288MILE7Xx14vh71iqVyqVorOzk5aWlolaLpejvb1dFzGV6WrDMgr3QEQ3TpmYAlk8FXIx/B1L9UokEgwPD1NXVzdRKxQK1NfX6yK7MmnMvYrceeedvPrqq9x5551xtyICQDKZJJ/PT6rl83mSyWRMHVUHhXtAGhoaePrpp1m9ejVPP/00DQ0NcbckQiaTIZ1Ok8vlKBQK5HI50um0pkJWmKZSBOQTn/gEfX19mBkNDQ185CMf4ejRo3G3JVVufMpj8UVMHR0dmgpZYRpzD0RtbS1vvvkmDzzwAF/4whc4ePAgX/rSl6ipqdEyBCKB0ph7Fbjrrrtwd+677z4aGhq47777cHfuuuuuuFsTkRgo3AMyPmPmSq9FpHrMGO5m1mNmL5tZX1Htr83sP83sOTP7ppmtKtq338zOmtl/mdm2SjUukx08eJBVq1Zx4sQJLl++zIkTJ1i1ahUHDx6MuzURicFsztwfBW6ZUvsOkHL39wP/DewHMLPrgduBLdF7/sbMtCzhAhgZGeHxxx+ftGb2448/rvF2kSo1Y7i7+7PAhSm1f3H38dQ4CayPtncAX3f3S+7+I+AscOM89itXMfVqP139J1K95mPMfTfwdLS9DnixaN9AVJMKa2xsZP/+/Tz00EO88cYbPPTQQ+zfv5/Gxsa4WxMhm82SSqVIJBKkUimtK7MAygp3M8sAI8DjJby3zcx6zax3cHCwnDYE6OrqYsWKFezbt4+Ghgb27dvHihUr6Orqirs1qXJaOCwm7j7jA9gI9E2pfQ74d2BFUW0/sL/o9XHgN2f6+R/60IdcynfkyBHfsmWL19TU+JYtW/zIkSNxtyTiW7Zs8RMnTkyqnThxwrds2RJTR+EAev0KuTqri5jMbCPwbXdPRa9vAR4CftfdB4uO2wIcYWyc/d3AM8Amd7/q6kC6iEkkXFo4rHLKuojJzLKMnaG/z8wGzCwNdAG/BHzHzL5nZgcB3P2HwBPA88A/A3fPFOwiEjYtHBaPGdeWcffpFoDovsrxHUBHOU2JSDjGFw7r7u6mubmZfD5POp2mo0MxUUlaOExEKkoLh8VDC4eJiCxRWjhMRKTKKNxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAKkcBcRCdCM4W5mPWb2spn1FdUazew7ZnYmel4d1c3MHjGzs2b2nJndUMnmZbJsNksqlSKRSJBKpchms3G3JCIxmc2Z+6PALVNq+4Bn3H0T8Ez0GmA7sCl6tAFfm582ZSbZbJZMJkNnZyfDw8N0dnaSyWQU8CJVasZwd/dngQtTyjuAw9H2YWBnUf0xH3MSWGVm75qvZuXKOjo66O7upqWlhbq6OlpaWuju7qajoyPu1kQkBqWOub/T3V+Ktn8CvDPaXge8WHTcQFR7CzNrM7NeM+sdHBwssQ0Z19/fT3Nz86Rac3Mz/f39MXUkInEq+wtVd3fAS3jfIXdvcvemtWvXlttG1Usmk+Tz+Um1fD5PMpmMqSMRiVOp4f7T8eGW6PnlqH4euKbouPVRTSosk8mQTqfJ5XIUCgVyuRzpdJpMJhN3ayISg9oS33cMuAP4SvT8raL6HjP7OvAR4LWi4RupoNbWVgDa29vp7+8nmUzS0dExUReR6mJjoypXOcAsC9wErAF+CtwPPAk8AVwLnAM+5e4XzMyALsZm17wBfN7de2dqoqmpyXt7ZzxMRESKmNlpd2+abt+MZ+7ufqVTv49Oc6wDd8+tPRERmW+6QlVEJEAKdxGRACncRUQCpHAXEQmQwl1EJEAKdxGRACncRUQCpHAXEQmQwl1EJEAKdxGRACncRUQCpHAXEQmQwl1EJEAKdxGRACncRUQCpHAXEQmQwl1EJEAKdxGRACncRUQCpHAXEQlQWeFuZvea2Q/NrM/MsmZWb2bvMbNTZnbWzI6a2bL5alZERGan5HA3s3XAnwBN7p4CEsDtwF8BD7v7dcArQHo+GhURkdkrd1imFnibmdUCK4CXgJuBf4j2HwZ2lvlriIjIHJUc7u5+HngA+DFjof4acBp41d1HosMGgHXlNikiInNTzrDMamAH8B7g3UADcMsc3t9mZr1m1js4OFhqGyIiMo1yhmU+BvzI3QfdvQB8A/htYFU0TAOwHjg/3Zvd/ZC7N7l709q1a8toQ0REpion3H8MbDWzFWZmwEeB54Ec8IfRMXcA3yqvRRERmatyxtxPMfbF6XeBH0Q/6xDwZeBPzews8A6gex76FBGROShrtoy73+/uv+buKXf/rLtfcvf/cfcb3f06d/8jd780X82KyNKUzWZJpVIkEglSqRTZbDbuloJXO/MhIiKly2azZDIZuru7aW5uJp/Pk06PXf7S2toac3fhMnePuweampq8t7c37jZEpAJSqRSdnZ20tLRM1HK5HO3t7fT19cXY2dJnZqfdvWnafQp3EamkRCLB8PAwdXV1E7VCoUB9fT2jo6Mxdrb0XS3ctXCYiFRUMpkkn89PquXzeZLJZEwdVQeFu4hUVCaTIZ1Ok8vlKBQK5HI50uk0mUwm7taCpi9URaSixr80bW9vp7+/n2QySUdHh75MrTCNuYuILFEacxcRqTIKdxGRACncRUQCpHAXEQmQwl1EJECLYraMmQ0C5+LuIyBrgJ/F3YTINPTZnF8b3H3aG2IsinCX+WVmvVeaHiUSJ302F46GZUREAqRwFxEJkMI9TIfibkDkCvTZXCAacxcRCZDO3EVEAqRwFxEJkMI9IGY2ambfK3psjLsnETNzM/u7ote1ZjZoZt+Os6/QaT33sPzC3T8QdxMiUwwBKTN7m7v/Avg94HzMPQVPZ+4ishCeAv4g2m4FsjH2UhUU7mF5W9GQzDfjbkakyNeB282sHng/cCrmfoKnYZmwaFhGFiV3fy76DqiVsbN4qTCFu4gslGPAA8BNwDvibSV8CncRWSg9wKvu/gMzuynuZkKncBeRBeHuA8AjcfdRLbT8gIhIgDRbRkQkQAp3EZEAKdxFRAKkcBcRCZDCXUQkQAp3EZEAKdxFRAL0/34UTnvLQxJpAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Example 2: A graph showing height distribution per sex\n",
    "fig, ax = plt.subplots()\n",
    "ax.boxplot([additional_fields.loc[covariates['sex'] == 0, 'height'].dropna(), \\\n",
    "        additional_fields.loc[covariates['sex'] == 1, 'height'].dropna()])\n",
    "ax.set_xticklabels(['F', 'M'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Working with genetic data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Raw markers (BED files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2019_12_29-14:09:45] /tmp/ukb_nadavb_chr17.bim: already exists.\n",
      "[2019_12_29-14:09:45] /tmp/ukb_nadavb_chr17.bed: already exists.\n",
      "[2019_12_29-14:09:45] /tmp/ukb_nadavb_chr17.fam: already exists.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Mapping files: 100%|██████████| 3/3 [00:02<00:00,  1.03it/s]\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>snp</th>\n",
       "      <th>cm</th>\n",
       "      <th>pos</th>\n",
       "      <th>a0</th>\n",
       "      <th>a1</th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>17</td>\n",
       "      <td>rs2294076</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6157</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>17</td>\n",
       "      <td>rs2396789</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8547</td>\n",
       "      <td>T</td>\n",
       "      <td>C</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>17</td>\n",
       "      <td>rs36068254</td>\n",
       "      <td>0.0</td>\n",
       "      <td>10583</td>\n",
       "      <td>A</td>\n",
       "      <td>G</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>17</td>\n",
       "      <td>rs6565705</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13905</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>17</td>\n",
       "      <td>rs2294075</td>\n",
       "      <td>0.0</td>\n",
       "      <td>14008</td>\n",
       "      <td>G</td>\n",
       "      <td>C</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  chrom         snp   cm    pos a0 a1  i\n",
       "0    17   rs2294076  0.0   6157  A  G  0\n",
       "1    17   rs2396789  0.0   8547  T  C  1\n",
       "2    17  rs36068254  0.0  10583  A  G  2\n",
       "3    17   rs6565705  0.0  13905  G  A  3\n",
       "4    17   rs2294075  0.0  14008  G  C  4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28835 total variants.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>fid</th>\n",
       "      <th>iid</th>\n",
       "      <th>father</th>\n",
       "      <th>mother</th>\n",
       "      <th>gender</th>\n",
       "      <th>trait</th>\n",
       "      <th>i</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1509492</td>\n",
       "      <td>5094518</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2042242</td>\n",
       "      <td>4066735</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3028220</td>\n",
       "      <td>2815615</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5193350</td>\n",
       "      <td>2632837</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4554111</td>\n",
       "      <td>2235304</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       fid      iid father mother gender  trait  i\n",
       "0  1509492  5094518      0      0      2    NaN  0\n",
       "1  2042242  4066735      0      0      1    NaN  1\n",
       "2  3028220  2815615      0      0      1    NaN  2\n",
       "3  5193350  2632837      0      0      2    NaN  3\n",
       "4  4554111  2235304      0      0      2    NaN  4"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 488377 records (shuffled).\n"
     ]
    }
   ],
   "source": [
    "# Load the genetic data for chr17.\n",
    "bim, fam, G = get_chrom_raw_marker_data('17')\n",
    "\n",
    "# 'bim' is a dataframe containing information about the variants.\n",
    "display(bim.head())\n",
    "print('%d total variants.' % len(bim))\n",
    "\n",
    "# 'fam' is a dataframe containing information about the samples (specifically, we need it to map the sample IDs).\n",
    "assert (fam['i'] == np.arange(len(fam))).all()\n",
    "summarize(fam)\n",
    "\n",
    "# G is the genotyping matrix (see usage below)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    128379\n",
       "1    234670\n",
       "2    202806\n",
       "3    344574\n",
       "4     64808\n",
       "Name: eid, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 274833 records (shuffled).\n"
     ]
    }
   ],
   "source": [
    "# In order to query the genotypes corresponding to samples in the phenotypic dataset we've parsed, we create a series\n",
    "# mapping each sample_index (i.e. the index shared by 'eid', 'additional_fields', 'covariates' etc.) to its correspondig \n",
    "# index within the G matrix. This is done by creating a series mapping between the 'iid' (= eid) to the 'i' columns in the\n",
    "# 'fam' dataframe. Note that the column 'i' provides the index within G of each sample.\n",
    "genotyping_index = eid.map(fam.astype({'iid': int}).set_index('iid')['i'])\n",
    "summarize(genotyping_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Variant 5/5\r"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chrom</th>\n",
       "      <th>snp</th>\n",
       "      <th>cm</th>\n",
       "      <th>pos</th>\n",
       "      <th>a0</th>\n",
       "      <th>a1</th>\n",
       "      <th>i</th>\n",
       "      <th>height_pval</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>20242</th>\n",
       "      <td>17</td>\n",
       "      <td>rs4968433</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59500333</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>20242</td>\n",
       "      <td>1.120956e-15</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20243</th>\n",
       "      <td>17</td>\n",
       "      <td>rs878745</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59502319</td>\n",
       "      <td>T</td>\n",
       "      <td>C</td>\n",
       "      <td>20243</td>\n",
       "      <td>1.170154e-22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20244</th>\n",
       "      <td>17</td>\n",
       "      <td>rs9896616</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59516472</td>\n",
       "      <td>T</td>\n",
       "      <td>C</td>\n",
       "      <td>20244</td>\n",
       "      <td>3.063647e-09</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20245</th>\n",
       "      <td>17</td>\n",
       "      <td>rs62071328</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59517034</td>\n",
       "      <td>T</td>\n",
       "      <td>C</td>\n",
       "      <td>20245</td>\n",
       "      <td>1.196334e-01</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>20246</th>\n",
       "      <td>17</td>\n",
       "      <td>rs146944614</td>\n",
       "      <td>0.0</td>\n",
       "      <td>59518015</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>20246</td>\n",
       "      <td>2.971785e-02</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      chrom          snp   cm       pos a0 a1      i   height_pval\n",
       "20242    17    rs4968433  0.0  59500333  G  A  20242  1.120956e-15\n",
       "20243    17     rs878745  0.0  59502319  T  C  20243  1.170154e-22\n",
       "20244    17    rs9896616  0.0  59516472  T  C  20244  3.063647e-09\n",
       "20245    17   rs62071328  0.0  59517034  T  C  20245  1.196334e-01\n",
       "20246    17  rs146944614  0.0  59518015  G  A  20246  2.971785e-02"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import statsmodels.api as sm\n",
    "\n",
    "# As a simple example, we will run association tests for each of the the variants in the gonomic locus\n",
    "# chr17:59,500,000-59,520,000 against the height phenotype.\n",
    "relevant_variants = bim[bim['pos'].between(59500000, 59520000)].copy()\n",
    "\n",
    "pval_results = []\n",
    "\n",
    "for i, (_, variant) in enumerate(relevant_variants.iterrows()):\n",
    "    \n",
    "    print('Variant %d/%d' % (i + 1, len(relevant_variants)), end = '\\r')\n",
    "    \n",
    "    # variant['i'] gives the index of the variant, so taking this row in the G matrix will give the genotypes of all the\n",
    "    # UKBB samples for this variant. The values in the G matrix indicate the genotype of each individual for each variant.\n",
    "    # These values can be 0 (homozygous reference, i.e. no copies of the alternative allele), 1 (heterozygous, i.e. one\n",
    "    # copy of the alternative allele) or 2 (homozygous alternative, i.e. two copies of the alternative allele).\n",
    "    variant_genotypes = G[variant['i'], :].compute()\n",
    "    # We use the genotyping_index to retrieve only the relevant samples from the loaded genotypes (and in the right order),\n",
    "    # so we end up with a vector of genotypes aligned to our dataset.\n",
    "    sample_variant_genotypes = variant_genotypes[genotyping_index.values]\n",
    "    sample_variant_genotypes = pd.Series(sample_variant_genotypes, index = eid.index).rename('genotype')\n",
    "    \n",
    "    # We construct the relevant dataset for this variant. The exogenous variables (X) will be all the covariates, along\n",
    "    # with the genotypes (the variable we are really interested in) and the endogenous variable (y) will be the height.\n",
    "    mask = pd.notnull(sample_variant_genotypes) & pd.notnull(additional_fields['height'])\n",
    "    y = additional_fields.loc[mask, 'height']\n",
    "    X = pd.concat([covariates, sample_variant_genotypes], axis = 1).loc[mask]\n",
    "    \n",
    "    # We train a simple linear-regression model (OLS) to calculate a p-value for this variant's genotypes (as a variable\n",
    "    # in the OLS, after accounting for all other covariates).\n",
    "    model = sm.OLS(y, X)\n",
    "    variant_results = model.fit()\n",
    "    pval = variant_results.pvalues['genotype']\n",
    "    pval_results.append(pval)\n",
    "\n",
    "# We finalize and display the results (the p-value derived for any of the variants).\n",
    "relevant_variants['height_pval'] = pval_results\n",
    "display(relevant_variants)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imputation genetic data (BGEN files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>chromosome</th>\n",
       "      <th>position</th>\n",
       "      <th>rsid</th>\n",
       "      <th>number_of_alleles</th>\n",
       "      <th>allele1</th>\n",
       "      <th>allele2</th>\n",
       "      <th>file_start_position</th>\n",
       "      <th>size_in_bytes</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>16</td>\n",
       "      <td>60086</td>\n",
       "      <td>rs537499872</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>C</td>\n",
       "      <td>15609040</td>\n",
       "      <td>52348</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>16</td>\n",
       "      <td>60105</td>\n",
       "      <td>rs556230663</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>C</td>\n",
       "      <td>15661388</td>\n",
       "      <td>7586</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>16</td>\n",
       "      <td>60112</td>\n",
       "      <td>rs575737384</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>15668974</td>\n",
       "      <td>5894</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>16</td>\n",
       "      <td>60164</td>\n",
       "      <td>rs181534180</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>T</td>\n",
       "      <td>15674868</td>\n",
       "      <td>110212</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>16</td>\n",
       "      <td>60169</td>\n",
       "      <td>rs558005370</td>\n",
       "      <td>2</td>\n",
       "      <td>G</td>\n",
       "      <td>A</td>\n",
       "      <td>15785080</td>\n",
       "      <td>12271</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  chromosome  position         rsid  number_of_alleles allele1 allele2  \\\n",
       "0         16     60086  rs537499872                  2       G       C   \n",
       "1         16     60105  rs556230663                  2       G       C   \n",
       "2         16     60112  rs575737384                  2       G       A   \n",
       "3         16     60164  rs181534180                  2       G       T   \n",
       "4         16     60169  rs558005370                  2       G       A   \n",
       "\n",
       "   file_start_position  size_in_bytes  \n",
       "0             15609040          52348  \n",
       "1             15661388           7586  \n",
       "2             15668974           5894  \n",
       "3             15674868         110212  \n",
       "4             15785080          12271  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3089612 total variants.\n",
      "487409 samples with imputed variants on this chromosome.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    3314279\n",
       "1    4761525\n",
       "2    3651490\n",
       "3    1747469\n",
       "4    5360208\n",
       "Name: eid, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 487409 records (shuffled).\n"
     ]
    }
   ],
   "source": [
    "# We load the imputation data for chr16.\n",
    "chrom_imputation_reader = get_chrom_imputation_data('16')\n",
    "\n",
    "# All the imputed variants on the chromosome can be accessed through this dataframe. \n",
    "display(chrom_imputation_reader.variants.head())\n",
    "\n",
    "print('%d total variants.' % chrom_imputation_reader.n_variants)\n",
    "print('%d samples with imputed variants on this chromosome.' % chrom_imputation_reader.n_samples)\n",
    "\n",
    "# Provides the eid corresponding to each sample in the imputation reader.\n",
    "summarize(chrom_imputation_reader.sample_ids)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "274224 of the 274833 dataset samples have imputated variants on this chromosome.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    446523\n",
       "1    191227\n",
       "2    215707\n",
       "3    477158\n",
       "4    310393\n",
       "Name: eid, dtype: int64"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Displaying 5 of 274224 records (shuffled).\n"
     ]
    }
   ],
   "source": [
    "# In order to query the imputed genotypes corresponding to samples in the phenotypic dataset we've parsed, we create a\n",
    "# series mapping each sample_index (i.e. the index shared by 'eid', 'additional_fields', 'covariates' etc.) to its\n",
    "# correspondig index within chrom_imputation_reader. This is done by creating a series mapping between the 'sample_ids'\n",
    "# (= eid) to the index.\n",
    "imputation_index = eid.map(chrom_imputation_reader.sample_ids.reset_index().set_index('eid')['index'])\n",
    "imputation_mask = pd.notnull(imputation_index)\n",
    "print('%d of the %d dataset samples have imputated variants on this chromosome.' % (imputation_mask.sum(), \\\n",
    "        len(imputation_mask)))\n",
    "imputation_index = imputation_index[imputation_mask].astype(int)\n",
    "summarize(imputation_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "chromosome                      16\n",
       "position                  53830465\n",
       "rsid                     rs9930506\n",
       "number_of_alleles                2\n",
       "allele1                          A\n",
       "allele2                          G\n",
       "file_start_position    43160279967\n",
       "size_in_bytes               135527\n",
       "Name: 1583331, dtype: object"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "There are 186261 carriers of the variant.\n",
      "12415 cases, 261809 controls.\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cases</th>\n",
       "      <th>controls</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>carriers</th>\n",
       "      <td>8817</td>\n",
       "      <td>177444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>non_carriers</th>\n",
       "      <td>3598</td>\n",
       "      <td>84365</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              cases  controls\n",
       "carriers       8817    177444\n",
       "non_carriers   3598     84365"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Fisher's exact test results: OR = 1.17, p-value = 2.6e-14\n"
     ]
    }
   ],
   "source": [
    "from scipy.stats import fisher_exact\n",
    "\n",
    "'''\n",
    "As a simple example, we consider the association between the variant rs9930506 in the FTO gene on chr16 to obesity (ICD-10\n",
    "code E66). According to SNPedia, there should be an association between them (https://www.snpedia.com/index.php/Rs9930506).\n",
    "'''\n",
    "\n",
    "# We query the record of the variant we look for (given by rsid).\n",
    "(_, variant_record), = chrom_imputation_reader.variants[chrom_imputation_reader.variants['rsid'] == 'rs9930506'].iterrows()\n",
    "display(variant_record)\n",
    "\n",
    "# We read the genotyping for the variant by providing the variant index (which is the name of the relevant row in the\n",
    "# chrom_imputation_reader.variants dataframe).\n",
    "variant_genotyping_probs = chrom_imputation_reader.read_variant_probs(variant_record.name)\n",
    "# We use imputation_index to retrieve only the relevant samples from the loaded genotypes (and in the right order),\n",
    "# so we end up with a vector of genotypes aligned to our dataset.\n",
    "sample_variant_genotyping_probs = variant_genotyping_probs[imputation_index.values, :]\n",
    "# The variant's genotyping is given as an Nx3 matrix, where N is the number of samples. Each row in the matrix\n",
    "# represents the genotyping of a specific individual. Since imputation is probabalistic, the genotyping is described as\n",
    "# three probabilities for having 0, 1 or 2 copies of the alternative allele, respectively (these are the same options\n",
    "# for the raw markers in the BED files, as described above).\n",
    "# For our purpose, we define a sample to be a carrier of the variant if there's at least 50% chance of it having any copy\n",
    "# of the alternative allele (under an assumed dominant effect), i.e. if the probability for 0 copies is smaller than 0.5.\n",
    "carrier_mask = (sample_variant_genotyping_probs[:, 0] <= 0.5)\n",
    "print('There are %d carriers of the variant.' % carrier_mask.sum())\n",
    "\n",
    "# We extract the node with ICD-10 coding E66, which stands for obesity.\n",
    "(_, ICD10_node), = ICD10_tree[ICD10_tree['coding'] == 'E66'].iterrows()\n",
    "# We create a mask vector, aligned to the samples used in imputation data, that would determine whether each sample is\n",
    "# inflicted with obesity.\n",
    "case_mask = pd.Series([(sample_index in ICD10_node['samples']) for sample_index in imputation_index.index], index = \\\n",
    "        imputation_index.index)\n",
    "control_mask = ~case_mask\n",
    "print('%d cases, %d controls.' % (case_mask.sum(), control_mask.sum()))\n",
    "\n",
    "# We construct a contingency table counting the number of sample with each combination of being either a carrier or\n",
    "# non-carrier of the allele, and being either of the case or the control group.\n",
    "contingency_table = pd.DataFrame([\n",
    "    [(carrier_mask & case_mask).sum(), (carrier_mask & control_mask).sum()],\n",
    "    [((~carrier_mask) & case_mask).sum(), ((~carrier_mask) & control_mask).sum()],\n",
    "], index = ['carriers', 'non_carriers'], columns = ['cases', 'controls'])\n",
    "display(contingency_table)\n",
    "\n",
    "# We run Fisher's exact test to test the reported association.\n",
    "print('Fisher\\'s exact test results: OR = %.2f, p-value = %.2g' % fisher_exact(contingency_table))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Tip:\n",
    "\n",
    "It is recommended to subscribe to the UKB-GENETICS@JISCMAIL.AC.UK mailing list (https://www.jiscmail.ac.uk/cgi-bin/webadmin?A0=UKB-GENETICS), where you can see the questions, answers and discussion of other researchers working with the UKBB genetic data, and also post your own questions regarding the dataset."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}