Vectorized SGD.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3e0e53a58f6444269a560fddfa1c0eac",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import random\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook, tqdm\n",
    "tqdm_notebook().pandas()\n",
    "import pickle\n",
    "# from pandarallel import pandarallel # for linux and mac\n",
    "# pandarallel.initialize()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = pd.read_csv('results.csv')\n",
    "results.loc[:, 'mask'] = results.loc[:, 'mask'].str.replace('X', '')\n",
    "roster = pd.read_csv('roster.csv')#.set_index('idplayer')\n",
    "tournaments = pd.read_csv('tournaments.csv').set_index('idtournament')\n",
    "tournaments = tournaments.loc[~tournaments.questions_total.isnull(), :]\n",
    "common_tours = list(set(tournaments.index).intersection(set(results.idtournament)))\n",
    "results = results.loc[results.idtournament.isin(common_tours), :]\n",
    "tournaments = tournaments.loc[common_tours, :]\n",
    "\n",
    "q_total = pd.DataFrame(results.groupby('idtournament').first()\\\n",
    "                       .loc[:, 'mask'].str.len()).dropna().rename(columns={'mask': 'questions_total'})\n",
    "tournaments = pd.merge(tournaments.drop(columns=['questions_total']), q_total, left_index=True, right_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "ineligible_tournaments = [1521, 1526, 1531, 1534, 1780, 1817, 1849, 1854, 1858, 1885, 1893, 1902, 1925, 1926, 1928,\n",
    "                          1931, 1960, 2025, 2040, 2077, 2087, 2128, 2147, 2151, 2166, 2183, 2291, 2345, 2346, 2472,\n",
    "                          2479, 2512, 2521, 2578, 2611, 2707, 2732, 2763, 2783, 2833, 3130, 3162, 3231, 3510, 3589,\n",
    "                          3590, 3617, 4011, 4169, 4375, 4469, 4481, 4497, 4713, 4714, 4816, 4860, 4881, 4913, 4946,\n",
    "                          4969, 4983, 4984, 4989, 4990, 4992, 5032, 5054, 5097, 5102, 5106, 5121, 5126, 5160, 5178,\n",
    "                          5210, 5215, 5221, 5239, 5241, 5247, 5299, 5310,\n",
    "                          # nan tours\n",
    "                          1954, 1989, 1970, 1810, 1917, 2373, 2219, 1969,  314, 1824, 2102, 2241, 2249, 2014, 2220,\n",
    "                          2363, 2703, 2234, 2417, 2171, 2126, 1900, 2013, 2394, 1792, 1689, 1590, 1987, 1895, 1991,\n",
    "                          2119, 1951, 1899,1898, 2224, 1976, 2100\n",
    "                         ]\n",
    "results = results.loc[~results.idtournament.isin(ineligible_tournaments), :]\n",
    "results.loc[:, 'mask'] = results.loc[:, 'mask'].apply(lambda x: list(str(x)))\n",
    "roster = roster.loc[~roster.idtournament.isin(ineligible_tournaments), :]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "results.loc[:, 'players'] = results.progress_apply(lambda row: np.sort(np.array(roster.loc[(roster.idteam == row['idteam']) & \n",
    "                                           (roster.idtournament == row['idtournament']), 'idplayer'].tolist())), axis=1)\n",
    "results = results.loc[results.players.str.len() != 0, :]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "results = results.loc[results.questions_total != 0, :]\n",
    "results.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "READ=True\n",
    "if READ:\n",
    "    with open('results.pickle', 'rb') as f:\n",
    "        results = pickle.load(f)\n",
    "else:\n",
    "    with open('results.pickle', 'wb') as f:\n",
    "        pickle.dump(results, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['idteam', 'current_name', 'base_name', 'position', 'questions_total',\n",
       "       'mask', 'bonus_a', 'bonus_b', 'tech_rating', 'predicted_position',\n",
       "       'd_bonus_a', 'd_bonus_b', 'd_diff_bonus', 'included_in_rating',\n",
       "       'idtournament', 'diff_bonus', 'players'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def explode(df, lst_cols, fill_value='', preserve_index=False):\n",
    "    # make sure `lst_cols` is list-alike\n",
    "    if (lst_cols is not None\n",
    "        and len(lst_cols) > 0\n",
    "        and not isinstance(lst_cols, (list, tuple, np.ndarray, pd.Series))):\n",
    "        lst_cols = [lst_cols]\n",
    "    # all columns except `lst_cols`\n",
    "    idx_cols = df.columns.difference(lst_cols)\n",
    "    # calculate lengths of lists\n",
    "    lens = df[lst_cols[0]].str.len()\n",
    "    # preserve original index values    \n",
    "    idx = np.repeat(df.index.values, lens)\n",
    "    orders = [num for range_len in lens for num in range(range_len)]\n",
    "    # create \"exploded\" DF\n",
    "    res = (pd.DataFrame({\n",
    "                col:np.repeat(df[col].values, lens)\n",
    "                for col in idx_cols},\n",
    "                index=idx)\n",
    "             .assign(**{col:np.concatenate(df.loc[lens>0, col].values)\n",
    "                            for col in lst_cols}))\n",
    "    # append those rows that have empty lists\n",
    "    if (lens == 0).any():\n",
    "        # at least one list in cells is empty\n",
    "        res = (res.append(df.loc[lens==0, idx_cols], sort=False)\n",
    "                  .fillna(fill_value))\n",
    "    # revert the original index order\n",
    "    res = res.sort_index()\n",
    "    # reset index if requested\n",
    "    if not preserve_index:        \n",
    "        res = res.reset_index(drop=True)\n",
    "    \n",
    "    res.loc[:, 'q_numb'] = orders\n",
    "    return res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idteam</th>\n",
       "      <th>idtournament</th>\n",
       "      <th>players</th>\n",
       "      <th>q_taken</th>\n",
       "      <th>q_numb</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>209</td>\n",
       "      <td>3236</td>\n",
       "      <td>[5974, 7474, 13601, 15624, 19298, 32017]</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   idteam  idtournament                                   players q_taken  \\\n",
       "0     209          3236  [5974, 7474, 13601, 15624, 19298, 32017]       1   \n",
       "\n",
       "   q_numb  \n",
       "0       0  "
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp = explode(results.loc[:, ['idteam', 'idtournament', 'mask', 'players']], ['mask'], fill_value='', preserve_index=False)\\\n",
    "            .rename(columns={'mask': 'q_taken'})\n",
    "exp.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.7 s ± 0 ns per loop (mean ± std. dev. of 1 run, 1 loop each)\n"
     ]
    }
   ],
   "source": [
    "%%timeit -n1 -r1\n",
    "# exp.loc[:, 'q_id_old'] = exp.idtournament.map(str) + \"_\" + exp.idteam.map(str) + \"_\" + exp.q_numb.map(str)\n",
    "# exp.loc[:, 'q_id'] = (exp.idtournament.map(str) + \"0000\" + exp.idteam.map(str) + \"0000\" + exp.q_numb.map(str)).map(int)\n",
    "exp.loc[:, 'q_id'] = exp.idtournament * 1000 + exp.q_numb\n",
    "exp.loc[:, 'qt_id'] = exp.idtournament * 1000000000 + exp.idteam * 1000 + exp.q_numb\n",
    "exp['q_taken'] = exp.q_taken.astype(int)\n",
    "# len(exp.q_id.unique()) == len(exp.q_id_old.unique()) check for losses"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idteam</th>\n",
       "      <th>idtournament</th>\n",
       "      <th>players</th>\n",
       "      <th>q_taken</th>\n",
       "      <th>q_numb</th>\n",
       "      <th>q_id</th>\n",
       "      <th>qt_id</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>209</td>\n",
       "      <td>3236</td>\n",
       "      <td>[5974, 7474, 13601, 15624, 19298, 32017]</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>3236000</td>\n",
       "      <td>3236000209000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   idteam  idtournament                                   players  q_taken  \\\n",
       "0     209          3236  [5974, 7474, 13601, 15624, 19298, 32017]        1   \n",
       "\n",
       "   q_numb     q_id          qt_id  \n",
       "0       0  3236000  3236000209000  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "exp.head(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train, test = train_test_split(exp, test_size=0.3, random_state=42)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Players skills and question difficulties as numpy arrays\n",
    "https://stackoverflow.com/questions/36652533/looking-up-large-sets-of-keys-dictionary-vs-numpy-array"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "LOAD = True\n",
    "if LOAD:\n",
    "    with open('skills.pickle', 'rb') as f:\n",
    "        skills = pickle.load(f)\n",
    "    with open('difficulties.pickle', 'rb') as f:\n",
    "        difficulties = pickle.load(f)\n",
    "else:\n",
    "    skill_ids = roster.sort_values('idplayer').idplayer.unique()\n",
    "    skills = np.array(list(zip(skill_ids, np.random.uniform(0,1, len(skill_ids))))   )\n",
    "\n",
    "    dif_ids = np.sort(exp.q_id.unique())\n",
    "    difficulties = np.array(list(zip(dif_ids, np.random.uniform(0,1, len(dif_ids))))   )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def team_performance(players):\n",
    "#     row_idx = np.searchsorted(skills[:,0], key_set)[key_set.argsort()]\n",
    "#     row_idx = np.searchsorted(skills[:,0], players)\n",
    "#     values = skills[row_idx, 1]\n",
    "    return np.mean(\n",
    "        skills[np.searchsorted(skills[:,0], players), 1] \n",
    "    )\n",
    "def get_difficulties_by_q_id(q_ids):\n",
    "    return difficulties[np.searchsorted(difficulties[:,0], q_ids), 1]#[q_ids.argsort()]\n",
    "def sigmoid(x):\n",
    "    return 1 / (1 + pd.np.exp(-x))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Learning"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Model**<br>\n",
    "$p(y=1| S_{team}, \\theta) = \\sigma(S_{team} - \\theta)$ <br>\n",
    "$S_{team}$ - team skill<br>\n",
    "$S_{team} = S_{player\\_1} + S_{player\\_2} + ... + S_{player\\_N}$<br>\n",
    "$\\theta_j$ - difficulty of question j<br>\n",
    "y - team answer, 1 for correct, 2 for incorrect<br>\n",
    "$p(y_{hat}=y_{real}| S_{team}, \\theta) = \\sigma(S_{team} - \\theta)^{y_{real}} * (1-\\sigma(S_{team} - \\theta))^{(1-y_{real})}$<br>\n",
    "<br>\n",
    "$LikelyHood = \\frac{1}{N}\\prod_{i=1}^{N} p(y_{hat\\_i}=y_{real\\_i}| S_{team}, \\theta)$<br>\n",
    "$log(LikelyHood) = LL = \\frac{1}{N}\\sum_{i=1}^{N} log(p(y_{hat\\_i}=y_{real\\_i}| S_{team}, \\theta))=\n",
    "y_{real}*log(\\sigma(S_{team} - \\theta)) + (1-y_{real})* log(1-\\sigma(S_{team} - \\theta))$<br>\n",
    "<br>\n",
    "$-log(LikelyHood) \\rightarrow min$ w.r.t. $S_{player\\_i}$ and $\\theta_j$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Derivatives:**\n",
    "\n",
    "$\\sigma'(x) = \\sigma(x)*(1-\\sigma(x))$\n",
    "\n",
    "$\\frac{\\partial LL}{\\partial \\theta} = \\frac{y_{real}}{\\sigma(S_{team} - \\theta)} * \\sigma'(S_{team} - \\theta) * (-1) +\n",
    "\\frac{1-y_{real}}{1-\\sigma(S_{team} - \\theta)} * (-\\sigma'(S_{team} - \\theta)) * (-1) = \n",
    "-y_{real}*(1-\\sigma(S_{team} - \\theta)) + (1-y_{real})*\\sigma(S_{team} - \\theta)\n",
    "$<br>\n",
    "\n",
    "$\\frac{\\partial LL}{\\partial S_i} = \\frac{y_{real}}{\\sigma(S_{team} - \\theta)} * \\sigma'(S_{team} - \\theta) +\n",
    "\\frac{1-y_{real}}{1-\\sigma(S_{team} - \\theta)} * (-\\sigma'(S_{team} - \\theta)) = \n",
    "y_{real}*(1-\\sigma(S_{team} - \\theta)) - (1-y_{real})*\\sigma(S_{team} - \\theta)$<br>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Short derivatives:**\n",
    "\n",
    "$\\frac{\\partial -LL}{\\partial \\theta} = +y_{real}*(1-\\sigma(S_{team} - \\theta)) - (1-y_{real})*\\sigma(S_{team} - \\theta)\n",
    "$<br>\n",
    "\n",
    "$\\frac{\\partial -LL}{\\partial S_i} = -y_{real}*(1-\\sigma(S_{team} - \\theta)) + (1-y_{real})*\\sigma(S_{team} - \\theta)$<br>\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**Gradient update**<br>\n",
    "$S_{i\\_new} = S_{i\\_old} - learning\\_rate * \\frac{\\partial LL}{\\partial S_i}$<br>\n",
    "$\\theta_{new} = \\theta_{old} - learning\\_rate * \\frac{\\partial LL}{\\partial \\theta}$<br>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "def sgd(df, lr=0.01, epochs=10, batch_size=16):\n",
    "    exp = df.copy(deep=True)\n",
    "    for _ in tqdm_notebook(range(epochs)):\n",
    "        # get batch\n",
    "        batch = exp.sample(n=batch_size)\n",
    "        # calculate team performance\n",
    "        t_perf = batch['players'].apply(lambda members: team_performance(members))\n",
    "        # get question difficulty\n",
    "        q_dif = batch['q_id'].apply(lambda question: get_difficulties_by_q_id(question))\n",
    "        # calculate  result \n",
    "        team_sigmoid = sigmoid(t_perf - q_dif)\n",
    "        # update question difficulty\n",
    "        q_dif_update = lr * (batch['q_taken'] * (1-team_sigmoid) - (1-batch['q_taken'])*team_sigmoid)\n",
    "        difficulties[np.searchsorted(difficulties[:,0], batch['q_id']), 1] -= q_dif_update\n",
    "        # update player skills\n",
    "        skill_update = -(lr * (batch['q_taken'] * (1-team_sigmoid) - (1-batch['q_taken'])*team_sigmoid) /\n",
    "                     batch['players'].str.len())\n",
    "        for idx in range(batch_size):\n",
    "            skills[np.searchsorted(skills[:,0], batch['players'].iloc[idx]), 1] -= skill_update.iloc[idx]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4aedc447cc8447fa90a7a4ec8596640b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=2000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# sgd(df = train, lr=0.5, epochs=20000, batch_size=5000)\n",
    "# sgd(df = train, lr=0.4, epochs=20000, batch_size=5000)\n",
    "sgd(df = train, lr=0.4, epochs=2000, batch_size=5000)\n",
    "# sgd(df = train, lr=0.5, epochs=20000, batch_size=5000)\n",
    "# sgd(lr=0.05, epochs=2000, batch_size=5000)\n",
    "# epochs 20k, batch 5k ETA 4:20 hours\n",
    "\n",
    "# sgd(lr=0.05, epochs=50, batch_size=1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "DUMP=True\n",
    "if DUMP:\n",
    "    with open('skills.pickle', 'wb') as f:\n",
    "        pickle.dump(skills, f)\n",
    "    with open('difficulties.pickle', 'wb') as f:\n",
    "        pickle.dump(difficulties, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def add_model_prob(df):\n",
    "    df.loc[:, 't_sigm'] = df.progress_apply(lambda row: sigmoid(team_performance(row['players']) -\n",
    "                                                                get_difficulties_by_q_id(row['q_id']) ), axis=1)\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9df5569ab12491e9f305776e6912526",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=7184560), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\ivan.mishalkin\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:362: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[key] = _infer_fill_value(value)\n",
      "C:\\Users\\ivan.mishalkin\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\pandas\\core\\indexing.py:543: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy\n",
      "  self.obj[item] = s\n"
     ]
    }
   ],
   "source": [
    "train = add_model_prob(train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "def calculate_loss(df):\n",
    "    df.loc[:, 'one_loss'] = df['q_taken']*np.log(df['t_sigm']) + (1-df['q_taken'])*np.log(1-df['t_sigm'])\n",
    "    total_loss = -df['one_loss'].sum()\n",
    "    print(total_loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3345915.3719092156\n"
     ]
    }
   ],
   "source": [
    "calculate_loss(train)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## ROC AUC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0c02e935a8304bbd8e94d8faf7c64237",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3079098), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "test = add_model_prob(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.metrics import roc_auc_score, roc_curve, auc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.8344042269174063"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "roc_auc_score(test.q_taken, test.t_sigm)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "fpr, tpr, thresholds = roc_curve(test.q_taken, test.t_sigm)\n",
    "roc_auc = auc(fpr, tpr)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1f106c31da0>]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD8CAYAAACMwORRAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvOIA7rQAAHclJREFUeJzt3Xl4nWW57/HvnXlOh6Rj2qYtLdAWKJAWqMyTBZV6tojlbDyi3bIRQUW357DF7YDb7T5OHFEcuhWZLkBAwYKFqgy7iBRIGdq0pTSd0zFp0szzus8fK3jFkDSLdmW9a/h9ritX1vuup2vdT1fy69PnHR5zd0REJLmkBV2AiIhEn8JdRCQJKdxFRJKQwl1EJAkp3EVEkpDCXUQkCSncRUSSkMJdRCQJKdxFRJJQRlBvXFJS4uXl5UG9vYhIQlq7dm2du5cO1y6wcC8vL6eysjKotxcRSUhmtjOSdpqWERFJQgp3EZEkpHAXEUlCCncRkSSkcBcRSULDhruZ3WVmB82saojnzczuMLNqM1tnZqdFv0wREXkvIhm53w0sPsLzlwGz+r6uA3527GWJiMixGPY8d3dfbWblR2iyBLjXw+v1rTGzUWY20d33RalGEZG44u509Ybo7nW6e0J0h0L09Do9veH9vSGnN+SEPPy9953vIScUcuZPHUVe1sheZhSNV58M7O63XdO3713hbmbXER7dM3Xq1Ci8tYhIWHdviNbOHlq7eml753tXD+1dvbR399Le1UtH9zuPQ3T0hPd19vTS0R2iozv8fGdPqO+rl87u8OOuvu2untDfQj0z3chMT+v7MjLS0sjo25eRZqSnGWnW9z3NwvvMSEuDH1w1PyHC3QbZN+iq2+6+HFgOUFFRoZW5RQQIj4RbOns43NZNY3s3h9u6aeropqm9m+aOHpo6+r63d9PU0UNzRzctnT3hr44emjt76A05+Vnp5GVlkJ+dTkF2BjmZ6eRlpZOblf63xzkZ4e2C7AxKCrLJyUwjJyP8fHZGGjmZ6eRkppGVkUZ2RnhfdmYaWelpZGemk5luZKWnYTZY9MWPaIR7DTCl33YZsDcKrysiCaqtq4dDLV3UtXRyqKWL+tYuDrV2cbgt/Li+tYuGti4a+sK8qb2brIw0RudlUZSbyei8TIpyMinKzaAwJ/x4ypg8inIyKMrNpDA7vL8gJxzkRTmZZGfEf+DGUjTCfQVwo5k9BJwBNGq+XST5uDuN7d0caOrkYHMH+xs7ONjcycGmDmpbOqlr7uJgc3hfb8gpKchmbEEWJQXZjM7LYmxBFqPzspheks+Y/CzG5GcxKi+LUX1BnpWhM7OjadhwN7MHgfOBEjOrAb4OZAK4+8+BlcDlQDXQBnxypIoVkZHh7tS1dLH3cDv7GjvY3xj+vq+xg/1NHRxoCod5VkYa4wqzGVeYw4TiHMYVZTNtbD4V5WMoKcimtDCb8UXZFGRnaBQdsEjOlrl6mOcd+GzUKhKRqHN36lu72Fnfxq5Dbeyub2N3Qxs1De3sOdzO/sYOcrPSmTwql4nFOUwszmXiqBxOmFjI+KIcxhflMLE4Z8QPAkr06JMSSRKhkLOvqYOdda1sP9TKrkNt7KpvY0dfmKenGdPG5jF1TB5TxuQxf8poPnDyJMpG5zKpOJfcrPSguyBRpHAXSTCN7d1srW1h68EWtte1svNQG1trW9hxqJXi3Eymjclnekk+U8fmcflJEykfm8/UMXkU52UGXbrEkMJdJA65OweaOtlysJktB1qo7gvzrbUttHf1MqO0gJml+cwoLeDSueOZUTKTGaX55GfrV1rC9JMgErCWzh427Wti494m3trfzNsHmtlyoJnM9DRmjS9g1rhCjh9fyOXzJjJzXD4TinJ0sFKGpXAXiRF3Z29jBxv3NrFhbyOb9oXD/GBTJ7PHFzBnUhEnTChiyfxJzBpXwNiC7KBLlgSmcBcZAe7OnsPtVO1p4s2aw1TtaaRqTyPpaWnMnVTE3ElFfPDkSfzLpYVML8knI13neEt0KdxFoqCpo5s3dx/mjV2HWburgfU1jaSlGSdNLuakycVcu6iceZOLGV+UE3SpkiIU7iLvkbuzra6VtTsaWLuzgbW7GtjT0M68yUWcOnU0SxdM4Tv/cJLmxiVQCneRYXT3hqja00jljgZe2VHP2p0N5GamU1E+mtOnjebjZ03jhAmFmlqRuKJwFxmguzfEm7sPs2bbIV7eXs/ruw5TNjqXBeVj+ODJE7ltyVwmFucGXabIESncJeW5O28faOHF6jperK7j5e31TB2Tx1kzx/LxM6fx46tPZVReVtBlirwnCndJSQebO3jh7Tr+0hfo2ZlpnH1cCR8+dTLf++gpjMlXmEtiU7hLSgiFnPV7GvnzpgM8s+kgNQ1tLJpZwjmzS/jCxbOYNjY/6BJFokrhLkmrtbOHF7bU8exbB3j2rVqKczO4+MTxfHPJXE6dMkoHQCWpKdwlqRxq6eSPGw/wp40HeGV7PfOnjOLCE8Zx4wWzmDo2L+jyRGJG4S4Jb3d9G09X7WfVhv1sPtDMubNLWTJ/Erd/bD7FuboToqQmhbskpG21LTy5bh+rNuxnf2MHl8wZz40XHseZM8aSk6n7koso3CVh7K5vY+X6ffxh/T72NXbwgZMm8tUPzGFB+WjNn4sMoHCXuHagqYMn3tzLE+v2UVPfxqVzx/O/338CZ84Yo0AXOQKFu8Sd9q5eVm3Yz6Nra1hXc5hL507gi5fMZtHMsWQq0EUionCXuODuvLH7MA9X1rBy/T7mTxnFxxZM4ZefqNAcushRULhLoA61dPLY63t4dG0N7d29XFUxhVVfOJcJxbo1rsixULhLzLk7a3c2cN+anTz71kEuPnE8X//QXM6YPoa0NN0iVyQaFO4SM21dPTzx5l7u+etO2rp6uObMadx2xTyK83Quuki0KdxlxO061Mav/7qdx17fQ8W00Xx58fGcN6tUo3SREaRwlxHh7ryyvZ67XtzOK9vruWrBFJ686WzKRusWACKxoHCXqOruDfFU1X7+a/U2Wrt6uHZROT+8aj752fpRE4kl/cZJVLR29vBw5W5+9ZftTCrO5XMXzeKiE8Zp6kUkIAp3OSYNrV3c89IO7n1pJ2dMH8MdV5/KaVNHB12WSMpTuMtR2dfYzi9f2M6ja2t4/9zxPHL9WcwsLQi6LBHpE1G4m9li4EdAOvBLd//PAc9PBe4BRvW1ucXdV0a5VokDuw618fPVW/nDun1ceXoZT33+HCaN0mLRIvFm2HA3s3TgTuASoAZ41cxWuPvGfs2+Cjzs7j8zsznASqB8BOqVgNQ0tPHjZ6r548b9/OMZ03j2S+cxtiA76LJEZAiRjNwXAtXuvg3AzB4ClgD9w92Bor7HxcDeaBYpwTnY1MFPn9/K42/s4R/PmMrz/3KBLjoSSQCRhPtkYHe/7RrgjAFtvgH80cxuAvKBi6NSnQSmpbOH5au3ce9LO/jIaWX86ebzKC3USF0kUUQS7oOdy+YDtq8G7nb3H5jZWcB9ZjbP3UN/90Jm1wHXAUydOvVo6pUR1htyfvtaDd9ftZlFM8fyxI1nM2WMLjwSSTSRhHsNMKXfdhnvnnZZBiwGcPeXzCwHKAEO9m/k7suB5QAVFRUD/4GQgL1YXce3ntxIfnYGv/j46ZyqUxpFElYk4f4qMMvMpgN7gKXA/xzQZhdwEXC3mZ0I5AC10SxURs7u+ja+/YdNbNjXyFcuO5HF8yZgpouPRBLZsOHu7j1mdiOwivBpjne5+wYzuw2odPcVwJeA/zKzmwlP2Vzr7hqZx7mO7l6Wr97Gr1/cziffN53/t3S+FsYQSRIRnefed876ygH7vtbv8UbgfdEtTUaKu/NU1X6+/YdNnFxWzArNq4skHV2hmmKqD7bwzSc2cLCpk+9/9BTOmjk26JJEZAQo3FNER3cvdz5Xzf1rdvLZC47jE4vKtdi0SBJTuKeAl7Ye4iuPref48YU8/YVzGV+k9UlFkp3CPYk1dXTz7Sc38cKWWr5xxVwunTsh6JJEJEb0//Ik9fzmgyy+fTXp6cYfv3iegl0kxWjknmRaO3v4j5WbeH5zLd+98hTOnlUSdEkiEgCFexJZu7OeLz38JqdNHc1TXziHohzd4EskVSnck0B3b4g7ntnCg6/s5t8/PI/F8zQFI5LqFO4Jbnd9Gzc++Dqj8zJZ+fmzGVeoM2FEROGe0J6u2setj1XxmfNnsuzs6bofjIj8jcI9AXX29PKdlW/x500H+NW1C5g/ZVTQJYlInFG4J5g9h9u54f61jCvK4cmbzmZUXlbQJYlIHFK4J5AXq+v4/ENv8OlzpnPduTM0DSMiQ1K4JwB3564Xd/Cz57dyx9XzWTRT566LyJEp3ONcV0+Irzy2nqo9jTx2wyLdmldEIqJwj2OH27q4/v61FOVk8rsbFpGXpY9LRCKje8vEqV2H2viHn/6VeZOK+dk1pyvYReQ9UWLEobU76/nn+17j8xcdx8fPKg+6HBFJQAr3OLNy/T6++ngVP7jqFC44flzQ5YhIglK4x5F7X9rBT5/byr2fWsi8ycVBlyMiCUzhHgfcnR89s4XHXt/DI9efpTNiROSYKdwD5u5884mNvLqjnkevX0RpYXbQJYlIElC4BygUcm59vIq39jfxwKfPpDhX918XkehQuAckFHL+9Xfr2V7Xyn3LzqAgWx+FiESPEiUAvSHnlt+uY2d9G3d9coGCXUSiTqkSY6GQ839+u46ahjZ+fe0C8hXsIjIClCwx5O589fdV7DrUxt2fWqCrTkVkxOj2AzHi7vzb76t4a18Tv7q2QsEuIiNKCRMj//n0W6zf08T9yxZSmKOzYkRkZGnkHgM/e34rz246yK+vXaBgF5GY0Mh9hN2/ZicPvLKTR69fxJh8LYknIrER0cjdzBab2WYzqzazW4Zoc5WZbTSzDWb2QHTLTExPV+3jJ89Wc/+yMxhflBN0OSKSQoYduZtZOnAncAlQA7xqZivcfWO/NrOAfwXe5+4NZpbytzOs3FHPVx6r4t5PLWTa2PygyxGRFBPJyH0hUO3u29y9C3gIWDKgzaeBO929AcDdD0a3zMSyrbaF6+9/jR9edYru7igigYgk3CcDu/tt1/Tt6282MNvMXjSzNWa2eLAXMrPrzKzSzCpra2uPruI419jWzT/dU8mXLp3N+bofu4gEJJJwt0H2+YDtDGAWcD5wNfBLMxv1rj/kvtzdK9y9orS09L3WGvd6ekPc+OBrnHd8KVcvnBp0OSKSwiIJ9xpgSr/tMmDvIG1+7+7d7r4d2Ew47FPKv/9hE2bGrZefGHQpIpLiIgn3V4FZZjbdzLKApcCKAW0eBy4AMLMSwtM026JZaLx7pHI3q9+u5cdLTyUjXZcPiEiwhk0hd+8BbgRWAZuAh919g5ndZmZX9DVbBRwys43Ac8CX3f3QSBUdb9bVHOY7T73Fzz9+OsV5ukhJRIJn7gOnz2OjoqLCKysrA3nvaKpv7eJDP/4L//bBE1k8b2LQ5YhIkjOzte5eMVw7zR8cg1DIufk3b/DBkycq2EUkrijcj8Edz26hvbuXL7//+KBLERH5O7q3zFH6a3UdD7y8iydvOlsHUEUk7iiVjsKhlk5ufvgNfnDVKYzTPWNEJA4p3N8jd+fWx6r48PzJnDMr+S7EEpHkoHB/j1a8uZettS3cfMnsoEsRERmS5tzfg5qGNm57YiN3f3IhOZnpQZcjIjIkjdwjFAo5X35kHcvOmc5JZbrTo4jEN4V7hO55aQedPb3887kzgy5FRGRYmpaJwLbaFu54Zgu/u+F9pKcNdpNMEZH4opH7MEIh55bfreemC2cxvUQrKolIYlC4D+M3lbvp6gnxiUXlQZciIhIxTcscQW1zJ99ftZl7ly3UdIyIJBSN3I/gO09t4iOnlzF3ks6OEZHEonAfwqs76nlp6yE+d1HKLSglIklA4T6Int4Q//Z4FV+5/EQKsjVzJSKJR+E+iAdf3U1xbiYfPFn3aBeRxKRwH6CxvZsf/fltvv6huZjpIKqIJCaF+wA/eXYLF50wnjmTioIuRUTkqCnc+9lR18qja2v40vt1x0cRSWwK936+t2oz/3TODMYVagEOEUlsCvc+b+4+TOXOej75vvKgSxEROWYKd8KrK/3Hyk184eLZ5GXp1EcRSXwKd2D1ljrqWjr56OllQZciIhIVKR/u7s4P//Q2N18ym4z0lP/rEJEkkfJptnpLHa2dPVw+TxcsiUjySOlwd3d+8uwWbrrwONJ010cRSSIpHe5rttVT29zJB07SqF1EkktKh/tPntvCDRccp7l2EUk6KZtq62sa2Vbbyv84dXLQpYiIRF1E4W5mi81ss5lVm9ktR2h3pZm5mVVEr8SR8fP/3sqys6eTqVG7iCShYZPNzNKBO4HLgDnA1WY2Z5B2hcDngJejXWS07a5v469b61i6cGrQpYiIjIhIhq0LgWp33+buXcBDwJJB2n0L+C7QEcX6RsSv/rKdqxZM0UIcIpK0Ign3ycDufts1ffv+xsxOBaa4+5NRrG1ENHd089jre7h2UXnQpYiIjJhIwn2wE8D9b0+apQG3A18a9oXMrjOzSjOrrK2tjbzKKHqksoZzZpUwsTg3kPcXEYmFSMK9BpjSb7sM2NtvuxCYBzxvZjuAM4EVgx1Udffl7l7h7hWlpaVHX/VRCoWc+9bs1KhdRJJeJOH+KjDLzKabWRawFFjxzpPu3ujuJe5e7u7lwBrgCnevHJGKj8HqLbXkZqZz+rTRQZciIjKihg13d+8BbgRWAZuAh919g5ndZmZXjHSB0fTAy7u45sxpWhtVRJJeRKeLuPtKYOWAfV8bou35x15W9B1o6uDl7fXc/rH5QZciIjLiUuYKnkfX1nD5SRPI1+mPIpICUiLcQyHnoVd3sXSBLloSkdSQEuG+Ztsh8rMyOLmsOOhSRERiIiXC/bev7eHK08t0IFVEUkbSh3trZw9/3LifJfN190cRSR1JH+5PVe1nYfkYSguzgy5FRCRmkj7cf/daDR85vSzoMkREYiqpw/1gcwdVexq58IRxQZciIhJTSR3uT1ft58ITxpGTmR50KSIiMZX04X6ZFr8WkRSUtOFe39rF+ppGzpsd+7tPiogELWnDfdWG/Zw7u1RTMiKSkpI23J+u2s/ieROCLkNEJBBJGe6tnT2s3dnA+cdrSkZEUlNShvtfqus4uayYwpzMoEsREQlEUob7M5sOcMmc8UGXISISmKQLd3fnuc21XHC8LlwSkdSVdOG+cV8T+VnplJfkB12KiEhgki7c//vtWp3bLiIpL+nC/YW36zhnlsJdRFJbUoV7R3cvb9Yc5syZY4MuRUQkUEkV7q/tamD2+EIKtAi2iKS4pAr3NdvqOXOGRu0iIkkV7i9treMsTcmIiCRPuLd39bJhbxMLykcHXYqISOCSJtxf29XACRMKycvSfLuISNKE+8vb61k4XVMyIiKQROFeuaOeM6aPCboMEZG4kBTh3hty1tU0curUUUGXIiISF5Ii3LfWtlBSkMWovKygSxERiQtJEe5v7D7MyWUatYuIvCOicDezxWa22cyqzeyWQZ7/opltNLN1ZvaMmU2LfqlDW1dzmFOmKNxFRN4xbLibWTpwJ3AZMAe42szmDGj2OlDh7icDjwLfjXahR7KuppGTy4pj+ZYiInEtkpH7QqDa3be5exfwELCkfwN3f87d2/o21wBl0S1zaF09Id4+0MzcSUWxeksRkbgXSbhPBnb3267p2zeUZcBTgz1hZteZWaWZVdbW1kZe5RFsOdhM2eg8XbwkItJPJOFug+zzQRuaXQNUAN8b7Hl3X+7uFe5eUVoanXuub9jTxEmTNSUjItJfJMPdGmBKv+0yYO/ARmZ2MXArcJ67d0anvOFt2t/E8RMKY/V2IiIJIZKR+6vALDObbmZZwFJgRf8GZnYq8AvgCnc/GP0yh/bWvmZOULiLiPydYcPd3XuAG4FVwCbgYXffYGa3mdkVfc2+BxQAj5jZG2a2YoiXi7q3DzRz4kQdTBUR6S+io5DuvhJYOWDf1/o9vjjKdUWkrqWT7t4Q4wqzg3h7EZG4ldBXqL69v5njJxRiNtgxXxGR1JXQ4V5d28Ks8ZpvFxEZKKHDfVttKzNK8oMuQ0Qk7iR0uG+va6V8rMJdRGSghA73bXUtzChVuIuIDJSw4d7VE+JAYydlo/OCLkVEJO4kbLjXNLQxvjibrIyE7YKIyIhJ2GTcWd+m+XYRkSEkbLjXNLRTNjo36DJEROJSwob7noZ2Jo9SuIuIDCZhw33v4XYmKdxFRAaVsOG+r1HhLiIylAQO9w4mFOUEXYaISFxKyHB3d2qbOxmvcBcRGVRChntTew9Z6WnkZqUHXYqISFxKyHA/0NxBaZHu4S4iMpSEDPfa5k5KCxTuIiJDSchwr2vppFSrL4mIDClBw72LsflZQZchIhK3EjLc61s7GatpGRGRISVouHczWiN3EZEhJWS4N7V3Myo3M+gyRETiVkKGe2N7N0UKdxGRISVkuDd1dFOscBcRGVJChntzRw+FORlBlyEiErcSMtxbOnsoyFa4i4gMJSHDva2zh3yFu4jIkBIu3N2dtu5ecjN10zARkaEkXLh3dIfISk8jPc2CLkVEJG4lXLi3d/fqVr8iIsOIKNzNbLGZbTazajO7ZZDns83sN33Pv2xm5dEu9B0d3b3kZCjcRUSOZNhwN7N04E7gMmAOcLWZzRnQbBnQ4O7HAbcD/zfahb6jqydEdmbC/YdDRCSmIknJhUC1u29z9y7gIWDJgDZLgHv6Hj8KXGRmIzIp3tkTnnMXEZGhRZKSk4Hd/bZr+vYN2sbde4BGYGw0ChyoqydEVobCXUTkSCJJycFG4H4UbTCz68ys0swqa2trI6nvXSYU5/DZC447qj8rIpIqIgn3GmBKv+0yYO9QbcwsAygG6ge+kLsvd/cKd68oLS09qoJLC7O5/KSJR/VnRURSRSTh/iowy8ymm1kWsBRYMaDNCuATfY+vBJ5193eN3EVEJDaGvYbf3XvM7EZgFZAO3OXuG8zsNqDS3VcAvwLuM7NqwiP2pSNZtIiIHFlEN2hx95XAygH7vtbvcQfw0eiWJiIiR0unnYiIJCGFu4hIElK4i4gkIYW7iEgSUriLiCQhC+p0dDOrBXYe5R8vAeqiWE4iUJ9Tg/qcGo6lz9PcfdirQAML92NhZpXuXhF0HbGkPqcG9Tk1xKLPmpYREUlCCncRkSSUqOG+POgCAqA+pwb1OTWMeJ8Tcs5dRESOLFFH7iIicgRxHe7xtDB3rETQ5y+a2UYzW2dmz5jZtCDqjKbh+tyv3ZVm5maW8GdWRNJnM7uq77PeYGYPxLrGaIvgZ3uqmT1nZq/3/XxfHkSd0WJmd5nZQTOrGuJ5M7M7+v4+1pnZaVEtwN3j8ovw7YW3AjOALOBNYM6ANjcAP+97vBT4TdB1x6DPFwB5fY8/kwp97mtXCKwG1gAVQdcdg895FvA6MLpve1zQdcegz8uBz/Q9ngPsCLruY+zzucBpQNUQz18OPEV4JbszgZej+f7xPHKPq4W5Y2TYPrv7c+7e1re5hvDKWIksks8Z4FvAd4GOWBY3QiLp86eBO929AcDdD8a4xmiLpM8OFPU9LubdK74lFHdfzSAr0vWzBLjXw9YAo8wsasvMxXO4x9XC3DESSZ/7W0b4X/5ENmyfzexUYIq7PxnLwkZQJJ/zbGC2mb1oZmvMbHHMqhsZkfT5G8A1ZlZDeP2Im2JTWmDe6+/7exLRYh0BidrC3Akk4v6Y2TVABXDeiFY08o7YZzNLA24Hro1VQTEQyeecQXhq5nzC/zt7wczmufvhEa5tpETS56uBu939B2Z2FuHV3ea5e2jkywvEiOZXPI/co7YwdwKJpM+Y2cXArcAV7t4Zo9pGynB9LgTmAc+b2Q7Cc5MrEvygaqQ/279392533w5sJhz2iSqSPi8DHgZw95eAHML3YElWEf2+H614DvdUXJh72D73TVH8gnCwJ/o8LAzTZ3dvdPcSdy9393LCxxmucPfKYMqNikh+th8nfPAcMyshPE2zLaZVRlckfd4FXARgZicSDvfamFYZWyuA/9V31syZQKO774vaqwd9RHmYo82XA28TPsp+a9++2wj/ckP4w38EqAZeAWYEXXMM+vxn4ADwRt/XiqBrHuk+D2j7PAl+tkyEn7MBPwQ2AuuBpUHXHIM+zwFeJHwmzRvApUHXfIz9fRDYB3QTHqUvA64Hru/3Gd/Z9/exPto/17pCVUQkCcXztIyIiBwlhbuISBJSuIuIJCGFu4hIElK4i4gkIYW7iEgSUriLiCQhhbuISBL6/9B+JQOKITPfAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.plot(fpr, tpr, lw=1, alpha=1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Exploit"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def at_least_same_taken_prob(probabilities, taken):\n",
    "    ques_amount = len(probabilities)\n",
    "    prob_matrix = np.zeros(shape=(ques_amount+1, ques_amount+1))\n",
    "    prob_matrix[0, 0] = 1\n",
    "    for i in range(ques_amount+1):\n",
    "        for j in range(1, ques_amount+1):\n",
    "            prob_matrix[i, j] = prob_matrix[i, j-1] * (1 - probabilities[j-1]) +\\\n",
    "                                prob_matrix[i-1, j-1] * probabilities[j-1]\n",
    "    return prob_matrix[taken:, -1].sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "21528007bab14af7bc2ffb70e914f430",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=247452), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "result_tr = train.groupby(['idtournament', 'idteam'])\\\n",
    "                 .progress_apply(lambda row: at_least_same_taken_prob(row['t_sigm'].tolist(), row['q_taken'].sum()))\\\n",
    "                 .reset_index()\\\n",
    "                 .rename(columns={0: 'prob'})\\\n",
    "                 .sort_values('prob')\\\n",
    "                 .reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "554c3769a38048cb88fcd0b17c8e2ec6",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=247445), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "result_te = test.groupby(['idtournament', 'idteam'])\\\n",
    "                .progress_apply(lambda row: at_least_same_taken_prob(row['t_sigm'].tolist(), row['q_taken'].sum()))\\\n",
    "                .reset_index()\\\n",
    "                .rename(columns={0: 'prob'})\\\n",
    "                .sort_values('prob')\\\n",
    "                .reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idtournament</th>\n",
       "      <th>idteam</th>\n",
       "      <th>prob</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2379</td>\n",
       "      <td>29548</td>\n",
       "      <td>1.399801e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2188</td>\n",
       "      <td>2919</td>\n",
       "      <td>2.158518e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2225</td>\n",
       "      <td>2919</td>\n",
       "      <td>3.852830e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>3327</td>\n",
       "      <td>6462</td>\n",
       "      <td>8.270742e-08</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1635</td>\n",
       "      <td>2919</td>\n",
       "      <td>1.096782e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>2695</td>\n",
       "      <td>6432</td>\n",
       "      <td>1.101028e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>5003</td>\n",
       "      <td>68862</td>\n",
       "      <td>1.218202e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>2111</td>\n",
       "      <td>4032</td>\n",
       "      <td>1.668413e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>2316</td>\n",
       "      <td>4967</td>\n",
       "      <td>1.778991e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>3370</td>\n",
       "      <td>42722</td>\n",
       "      <td>2.227831e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>2906</td>\n",
       "      <td>49984</td>\n",
       "      <td>4.492690e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>3003</td>\n",
       "      <td>8187</td>\n",
       "      <td>5.640080e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>2849</td>\n",
       "      <td>38064</td>\n",
       "      <td>6.532888e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>3329</td>\n",
       "      <td>44699</td>\n",
       "      <td>6.620794e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>4981</td>\n",
       "      <td>56211</td>\n",
       "      <td>6.675211e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>3451</td>\n",
       "      <td>42722</td>\n",
       "      <td>6.707433e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>4664</td>\n",
       "      <td>49682</td>\n",
       "      <td>6.919137e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>3411</td>\n",
       "      <td>43456</td>\n",
       "      <td>7.528551e-07</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>3459</td>\n",
       "      <td>38124</td>\n",
       "      <td>1.207706e-06</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>4434</td>\n",
       "      <td>10364</td>\n",
       "      <td>1.813839e-06</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "    idtournament  idteam          prob\n",
       "0           2379   29548  1.399801e-08\n",
       "1           2188    2919  2.158518e-08\n",
       "2           2225    2919  3.852830e-08\n",
       "3           3327    6462  8.270742e-08\n",
       "4           1635    2919  1.096782e-07\n",
       "5           2695    6432  1.101028e-07\n",
       "6           5003   68862  1.218202e-07\n",
       "7           2111    4032  1.668413e-07\n",
       "8           2316    4967  1.778991e-07\n",
       "9           3370   42722  2.227831e-07\n",
       "10          2906   49984  4.492690e-07\n",
       "11          3003    8187  5.640080e-07\n",
       "12          2849   38064  6.532888e-07\n",
       "13          3329   44699  6.620794e-07\n",
       "14          4981   56211  6.675211e-07\n",
       "15          3451   42722  6.707433e-07\n",
       "16          4664   49682  6.919137e-07\n",
       "17          3411   43456  7.528551e-07\n",
       "18          3459   38124  1.207706e-06\n",
       "19          4434   10364  1.813839e-06"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "result_tr.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "# zip(results.loc[results.questions_total == 0, ['idteam', 'idtournament']])\n",
    "# results.loc[results.questions_total == 0, 'idtournament'].tolist()\n",
    "res = pd.merge( result_tr, results.loc[results.questions_total != 0, ['idteam', 'idtournament']], on=['idteam', 'idtournament'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.iloc[1000:1010]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "def susp_team_res(team, tour):\n",
    "    susp_team = results.loc[(results.idteam == team) & (results.idtournament == tour), 'players'].item()\n",
    "    return results.loc[results.players.astype(str) == str(susp_team), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "team = 4967\n",
    "tour = 4385"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "p=0.3\n",
    "i=0\n",
    "tmp = res.loc[res.prob > p, :]\n",
    "tmp = tmp.reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.30003999139553356"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "i += 10\n",
    "team = tmp.idteam[i]\n",
    "tour = tmp.idtournament[i]\n",
    "tmp.prob[i]"
   ]
  },
  {
   "cell_type": "raw",
   "metadata": {},
   "source": [
    "team = 29548\n",
    "tour = 2379"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>idteam</th>\n",
       "      <th>current_name</th>\n",
       "      <th>base_name</th>\n",
       "      <th>position</th>\n",
       "      <th>questions_total</th>\n",
       "      <th>mask</th>\n",
       "      <th>bonus_a</th>\n",
       "      <th>bonus_b</th>\n",
       "      <th>tech_rating</th>\n",
       "      <th>predicted_position</th>\n",
       "      <th>d_bonus_a</th>\n",
       "      <th>d_bonus_b</th>\n",
       "      <th>d_diff_bonus</th>\n",
       "      <th>included_in_rating</th>\n",
       "      <th>idtournament</th>\n",
       "      <th>diff_bonus</th>\n",
       "      <th>players</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>63839</th>\n",
       "      <td>7939</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>32.0</td>\n",
       "      <td>22</td>\n",
       "      <td>[1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, ...</td>\n",
       "      <td>2288.0</td>\n",
       "      <td>177.0</td>\n",
       "      <td>2819.0</td>\n",
       "      <td>151.0</td>\n",
       "      <td>2288</td>\n",
       "      <td>1026</td>\n",
       "      <td>177.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3325</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[5380, 5517, 13642, 22044, 52047, 70416]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>70246</th>\n",
       "      <td>7939</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>120.0</td>\n",
       "      <td>27</td>\n",
       "      <td>[0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, ...</td>\n",
       "      <td>2236.0</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>2730.0</td>\n",
       "      <td>115.0</td>\n",
       "      <td>2236</td>\n",
       "      <td>555</td>\n",
       "      <td>-1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2913</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[5380, 5517, 13642, 22044, 52047, 70416]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100819</th>\n",
       "      <td>7939</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>154.0</td>\n",
       "      <td>14</td>\n",
       "      <td>[1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, ...</td>\n",
       "      <td>1206.0</td>\n",
       "      <td>-44.0</td>\n",
       "      <td>2177.0</td>\n",
       "      <td>91.0</td>\n",
       "      <td>1206</td>\n",
       "      <td>248</td>\n",
       "      <td>-44.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>2248</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[5380, 5517, 13642, 22044, 52047, 70416]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>176175</th>\n",
       "      <td>7939</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>172.5</td>\n",
       "      <td>20</td>\n",
       "      <td>[1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, ...</td>\n",
       "      <td>2249.0</td>\n",
       "      <td>-29.0</td>\n",
       "      <td>3218.0</td>\n",
       "      <td>105.0</td>\n",
       "      <td>2249</td>\n",
       "      <td>623</td>\n",
       "      <td>-29.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3327</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[5380, 5517, 13642, 22044, 52047, 70416]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>180796</th>\n",
       "      <td>7939</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>96.0</td>\n",
       "      <td>20</td>\n",
       "      <td>[1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, ...</td>\n",
       "      <td>2243.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>2700.0</td>\n",
       "      <td>102.0</td>\n",
       "      <td>2243</td>\n",
       "      <td>561</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3007</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[5380, 5517, 13642, 22044, 52047, 70416]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>181768</th>\n",
       "      <td>7939</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>Дежавю</td>\n",
       "      <td>154.0</td>\n",
       "      <td>17</td>\n",
       "      <td>[0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, ...</td>\n",
       "      <td>2150.0</td>\n",
       "      <td>-21.0</td>\n",
       "      <td>2736.0</td>\n",
       "      <td>103.0</td>\n",
       "      <td>2150</td>\n",
       "      <td>474</td>\n",
       "      <td>-21.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>3006</td>\n",
       "      <td>NaN</td>\n",
       "      <td>[5380, 5517, 13642, 22044, 52047, 70416]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "        idteam current_name base_name  position  questions_total  \\\n",
       "63839     7939       Дежавю    Дежавю      32.0               22   \n",
       "70246     7939       Дежавю    Дежавю     120.0               27   \n",
       "100819    7939       Дежавю    Дежавю     154.0               14   \n",
       "176175    7939       Дежавю    Дежавю     172.5               20   \n",
       "180796    7939       Дежавю    Дежавю      96.0               20   \n",
       "181768    7939       Дежавю    Дежавю     154.0               17   \n",
       "\n",
       "                                                     mask  bonus_a  bonus_b  \\\n",
       "63839   [1, 1, 0, 0, 1, 1, 1, 0, 1, 1, 1, 0, 1, 1, 0, ...   2288.0    177.0   \n",
       "70246   [0, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 0, ...   2236.0     -1.0   \n",
       "100819  [1, 0, 0, 1, 0, 0, 0, 0, 0, 0, 1, 0, 0, 1, 0, ...   1206.0    -44.0   \n",
       "176175  [1, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, 0, 0, 1, ...   2249.0    -29.0   \n",
       "180796  [1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 0, 1, 1, 0, ...   2243.0      4.0   \n",
       "181768  [0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 0, 0, ...   2150.0    -21.0   \n",
       "\n",
       "        tech_rating  predicted_position  d_bonus_a  d_bonus_b  d_diff_bonus  \\\n",
       "63839        2819.0               151.0       2288       1026         177.0   \n",
       "70246        2730.0               115.0       2236        555          -1.0   \n",
       "100819       2177.0                91.0       1206        248         -44.0   \n",
       "176175       3218.0               105.0       2249        623         -29.0   \n",
       "180796       2700.0               102.0       2243        561           4.0   \n",
       "181768       2736.0               103.0       2150        474         -21.0   \n",
       "\n",
       "        included_in_rating  idtournament  diff_bonus  \\\n",
       "63839                  1.0          3325         NaN   \n",
       "70246                  1.0          2913         NaN   \n",
       "100819                 1.0          2248         NaN   \n",
       "176175                 1.0          3327         NaN   \n",
       "180796                 1.0          3007         NaN   \n",
       "181768                 1.0          3006         NaN   \n",
       "\n",
       "                                         players  \n",
       "63839   [5380, 5517, 13642, 22044, 52047, 70416]  \n",
       "70246   [5380, 5517, 13642, 22044, 52047, 70416]  \n",
       "100819  [5380, 5517, 13642, 22044, 52047, 70416]  \n",
       "176175  [5380, 5517, 13642, 22044, 52047, 70416]  \n",
       "180796  [5380, 5517, 13642, 22044, 52047, 70416]  \n",
       "181768  [5380, 5517, 13642, 22044, 52047, 70416]  "
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "susp_res = susp_team_res(team, tour)\n",
    "susp_res"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tour"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "res.loc[(res.idteam == team) & (res.idtournament == tour), :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(susp_res['mask'].iloc[1])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results.loc[results.idtournament == tour, :]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_t_res = exp.loc[exp.idtournament == tour, :]. groupby('idteam').q_taken.apply(lambda x: sum(x) / len(x)).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_t_res.loc[grouped_t_res.idteam == team]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "grouped_t_res.q_taken.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "roster\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}