1a_active_learning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os.path\n",
    "import sys\n",
    "import django\n",
    "sys.path.append('/home/galm/software/django/tmv/BasicBrowser')\n",
    "sys.path.append('/home/max/software/django-tmv/tmv_mcc-apsis/BasicBrowser')\n",
    "os.environ.setdefault(\"DJANGO_SETTINGS_MODULE\", \"BasicBrowser.settings\")\n",
    "django.setup()\n",
    "import scoping\n",
    "from scoping.models import *\n",
    "from utils.text import *\n",
    "\n",
    "from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer, TfidfTransformer\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score, roc_auc_score, accuracy_score\n",
    "from sklearn.svm import SVC\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "\n",
    "import pickle\n",
    "import scipy.sparse\n",
    "\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(531125, 110)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>title</th>\n",
       "      <th>content</th>\n",
       "      <th>0 - relevant</th>\n",
       "      <th>10 - 3. Quantitative</th>\n",
       "      <th>10 - 4. Qualitative</th>\n",
       "      <th>11 - countries</th>\n",
       "      <th>12 - policy name</th>\n",
       "      <th>2 - 0. Mitigation</th>\n",
       "      <th>2 - 1. Adaptation</th>\n",
       "      <th>...</th>\n",
       "      <th>9 - 0. Ex-post</th>\n",
       "      <th>9 - 1. Ex-ante</th>\n",
       "      <th>9 - 2. Ex-ante (IAM)</th>\n",
       "      <th>INCLUDE</th>\n",
       "      <th>representative_sample</th>\n",
       "      <th>seen</th>\n",
       "      <th>wosarticle__de</th>\n",
       "      <th>wosarticle__wc</th>\n",
       "      <th>relevant</th>\n",
       "      <th>random</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3812405</td>\n",
       "      <td>South Asian summer monsoon projections constra...</td>\n",
       "      <td>A reliable projection of future South Asian su...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Multidisciplinary Sciences']</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>233883</td>\n",
       "      <td>Millennial to interannual climate variability ...</td>\n",
       "      <td>Climate change during the late Pleistocene is ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Geography, Physical; Geosciences, Multidisci...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1857568</td>\n",
       "      <td>Modeling of CO2 Adsorption on Activated Carbon...</td>\n",
       "      <td>Due to carbon dioxide role in global warming, ...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>['Engineering, Chemical; Materials Science, Mu...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>574396</td>\n",
       "      <td>The positive effects of bed material coating o...</td>\n",
       "      <td>The utilization of biomass for the substitutio...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>Biomass gasification; Bed material; Catalytic ...</td>\n",
       "      <td>['Energy &amp; Fuels; Engineering, Chemical']</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>545538</td>\n",
       "      <td>Numerical simulation and optimization of CO2 s...</td>\n",
       "      <td>With heightened concerns on CO2 emissions from...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>CO2 sequestration; Computational fluid dynamic...</td>\n",
       "      <td>['Computer Science, Interdisciplinary Applicat...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 110 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "        id                                              title  \\\n",
       "0  3812405  South Asian summer monsoon projections constra...   \n",
       "1   233883  Millennial to interannual climate variability ...   \n",
       "2  1857568  Modeling of CO2 Adsorption on Activated Carbon...   \n",
       "3   574396  The positive effects of bed material coating o...   \n",
       "4   545538  Numerical simulation and optimization of CO2 s...   \n",
       "\n",
       "                                             content  0 - relevant  \\\n",
       "0  A reliable projection of future South Asian su...           NaN   \n",
       "1  Climate change during the late Pleistocene is ...           NaN   \n",
       "2  Due to carbon dioxide role in global warming, ...           NaN   \n",
       "3  The utilization of biomass for the substitutio...           NaN   \n",
       "4  With heightened concerns on CO2 emissions from...           NaN   \n",
       "\n",
       "   10 - 3. Quantitative  10 - 4. Qualitative 11 - countries 12 - policy name  \\\n",
       "0                   NaN                  NaN            NaN              NaN   \n",
       "1                   NaN                  NaN            NaN              NaN   \n",
       "2                   NaN                  NaN            NaN              NaN   \n",
       "3                   NaN                  NaN            NaN              NaN   \n",
       "4                   NaN                  NaN            NaN              NaN   \n",
       "\n",
       "   2 - 0. Mitigation  2 - 1. Adaptation  ...  9 - 0. Ex-post  9 - 1. Ex-ante  \\\n",
       "0                NaN                NaN  ...             NaN             NaN   \n",
       "1                NaN                NaN  ...             NaN             NaN   \n",
       "2                NaN                NaN  ...             NaN             NaN   \n",
       "3                NaN                NaN  ...             NaN             NaN   \n",
       "4                NaN                NaN  ...             NaN             NaN   \n",
       "\n",
       "   9 - 2. Ex-ante (IAM)  INCLUDE  representative_sample  seen  \\\n",
       "0                   NaN      NaN                    NaN     0   \n",
       "1                   NaN      NaN                    NaN     0   \n",
       "2                   NaN      NaN                    NaN     0   \n",
       "3                   NaN      NaN                    NaN     0   \n",
       "4                   NaN      NaN                    NaN     0   \n",
       "\n",
       "                                      wosarticle__de  \\\n",
       "0                                                NaN   \n",
       "1                                                NaN   \n",
       "2                                                NaN   \n",
       "3  Biomass gasification; Bed material; Catalytic ...   \n",
       "4  CO2 sequestration; Computational fluid dynamic...   \n",
       "\n",
       "                                      wosarticle__wc  relevant  random  \n",
       "0                     ['Multidisciplinary Sciences']       0.0     0.0  \n",
       "1  ['Geography, Physical; Geosciences, Multidisci...       0.0     0.0  \n",
       "2  ['Engineering, Chemical; Materials Science, Mu...       0.0     0.0  \n",
       "3          ['Energy & Fuels; Engineering, Chemical']       0.0     0.0  \n",
       "4  ['Computer Science, Interdisciplinary Applicat...       0.0     0.0  \n",
       "\n",
       "[5 rows x 110 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seen_df = pd.read_csv('data/0_labelled_documents.csv')\n",
    "seen_df = seen_df[pd.notna(seen_df['INCLUDE'])]\n",
    "seen_df[\"seen\"] = 1\n",
    "unseen_df = pd.read_csv('data/0_unlabelled_documents.csv')\n",
    "df = (pd.concat([seen_df,unseen_df])\n",
    "      .sort_values('id')\n",
    "      .sample(frac=1, random_state=1)\n",
    "      .reset_index(drop=True)\n",
    ")\n",
    "\n",
    "seen_index = df[df['seen']==1].index\n",
    "unseen_index = df[df['seen']==0].index\n",
    "\n",
    "print(df.shape)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.axes._subplots.AxesSubplot at 0x7fd51c59e390>"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEMCAYAAAAoB2Y1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAPPElEQVR4nO3df6zdd13H8eeL1YEw2a/e1NEW73SNOE1gS1MmGCVUYRtolwgTVFaWmvpjIDjBVWKyoFG3oE4WybTSQRcJjEx0dSyQpWwxBld298PBNrA3G6NttvXCfiASZJO3f9xP3dnldl3vuT1nu5/nIzk538+P7/f7vk3zOt/7Od9zbqoKSVIfnjfuAiRJo2PoS1JHDH1J6oihL0kdMfQlqSPLxl3A01m+fHlNTk6OuwxJek659dZbv15VE/ONPatDf3JykqmpqXGXIUnPKUnuP9iYyzuS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktSRZ/Uncp8rJrd8etwlLClfveQN4y5BWrK80pekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOHDL0k1yZZH+SLw30nZDkhiS72/PxrT9JLk8yneTOJKcP7LOxzd+dZOOR+XEkSU/nmVzpfxQ4c07fFmBnVa0BdrY2wFnAmvbYDFwBsy8SwMXAK4F1wMUHXigkSaNzyNCvqn8FHp7TvQHY3ra3A+cM9F9Vs24GjktyEvB64IaqeriqHgFu4PtfSCRJR9hC1/RXVNUDbftBYEXbXgnsGZi3t/UdrF+SNEJDv5FbVQXUItQCQJLNSaaSTM3MzCzWYSVJLDz0H2rLNrTn/a1/H7B6YN6q1new/u9TVVuram1VrZ2YmFhgeZKk+Sw09HcAB+7A2QhcO9B/XruL5wzgsbYM9FngdUmOb2/gvq71SZJGaNmhJiT5OPAaYHmSvczehXMJ8Mkkm4D7gXPb9OuBs4Fp4NvA+QBV9XCSPwFuafP+uKrmvjksSTrCDhn6VfXWgwytn2duARcc5DhXAlceVnWSpEXlJ3IlqSOGviR1xNCXpI4Y+pLUkUO+kSvpuW1yy6fHXcKS8dVL3jDuEobmlb4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHRkq9JP8XpK7knwpyceTvCDJyUl2JZlOcnWSo9vc57f2dBufXIwfQJL0zC049JOsBH4XWFtVPwUcBbwFuBS4rKpOAR4BNrVdNgGPtP7L2jxJ0ggNu7yzDPjBJMuAFwIPAK8Frmnj24Fz2vaG1qaNr0+SIc8vSToMCw79qtoH/AXwNWbD/jHgVuDRqnqiTdsLrGzbK4E9bd8n2vwTF3p+SdLhG2Z553hmr95PBl4CvAg4c9iCkmxOMpVkamZmZtjDSZIGDLO88/PAfVU1U1WPA58CXg0c15Z7AFYB+9r2PmA1QBs/FvjG3INW1daqWltVaycmJoYoT5I01zCh/zXgjCQvbGvz64G7gRuBN7U5G4Fr2/aO1qaNf66qaojzS5IO0zBr+ruYfUP2NuCL7VhbgYuAC5NMM7tmv63tsg04sfVfCGwZom5J0gIsO/SUg6uqi4GL53TfC6ybZ+53gDcPcz5J0nD8RK4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkeGCv0kxyW5JsmXk9yT5KeTnJDkhiS72/PxbW6SXJ5kOsmdSU5fnB9BkvRMDXul/0HgM1X1MuDlwD3AFmBnVa0BdrY2wFnAmvbYDFwx5LklSYdpwaGf5FjgZ4FtAFX13ap6FNgAbG/TtgPntO0NwFU162bguCQnLbhySdJhG+ZK/2RgBvhIktuTfDjJi4AVVfVAm/MgsKJtrwT2DOy/t/U9RZLNSaaSTM3MzAxRniRprmFCfxlwOnBFVZ0G/DdPLuUAUFUF1OEctKq2VtXaqlo7MTExRHmSpLmGCf29wN6q2tXa1zD7IvDQgWWb9ry/je8DVg/sv6r1SZJGZMGhX1UPAnuS/HjrWg/cDewANra+jcC1bXsHcF67i+cM4LGBZSBJ0ggsG3L/dwIfS3I0cC9wPrMvJJ9Msgm4Hzi3zb0eOBuYBr7d5kqSRmio0K+qO4C18wytn2duARcMcz5J0nD8RK4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6MnToJzkqye1Jrmvtk5PsSjKd5OokR7f+57f2dBufHPbckqTDsxhX+u8C7hloXwpcVlWnAI8Am1r/JuCR1n9ZmydJGqGhQj/JKuANwIdbO8BrgWvalO3AOW17Q2vTxte3+ZKkERn2Sv+vgT8AvtfaJwKPVtUTrb0XWNm2VwJ7ANr4Y23+UyTZnGQqydTMzMyQ5UmSBi049JO8EdhfVbcuYj1U1daqWltVaycmJhbz0JLUvWVD7Ptq4JeSnA28AHgx8EHguCTL2tX8KmBfm78PWA3sTbIMOBb4xhDnlyQdpgVf6VfVH1bVqqqaBN4CfK6qfg24EXhTm7YRuLZt72ht2vjnqqoWen5J0uE7EvfpXwRcmGSa2TX7ba1/G3Bi678Q2HIEzi1JehrDLO/8v6q6Cbipbd8LrJtnzneANy/G+SRJC+MnciWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktSRBYd+ktVJbkxyd5K7kryr9Z+Q5IYku9vz8a0/SS5PMp3kziSnL9YPIUl6Zoa50n8C+P2qOhU4A7ggyanAFmBnVa0BdrY2wFnAmvbYDFwxxLklSQuw4NCvqgeq6ra2/V/APcBKYAOwvU3bDpzTtjcAV9Wsm4Hjkpy04MolSYdtUdb0k0wCpwG7gBVV9UAbehBY0bZXAnsGdtvb+uYea3OSqSRTMzMzi1GeJKkZOvSTHAP8I/Duqvrm4FhVFVCHc7yq2lpVa6tq7cTExLDlSZIGDBX6SX6A2cD/WFV9qnU/dGDZpj3vb/37gNUDu69qfZKkERnm7p0A24B7quqvBoZ2ABvb9kbg2oH+89pdPGcAjw0sA0mSRmDZEPu+Gngb8MUkd7S+9wGXAJ9Msgm4Hzi3jV0PnA1MA98Gzh/i3JKkBVhw6FfVvwE5yPD6eeYXcMFCzydJGp6fyJWkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSOGviR1xNCXpI4Y+pLUEUNfkjpi6EtSRwx9SeqIoS9JHTH0Jakjhr4kdcTQl6SOGPqS1BFDX5I6YuhLUkcMfUnqiKEvSR0x9CWpI4a+JHXE0Jekjhj6ktQRQ1+SOmLoS1JHRh76Sc5M8pUk00m2jPr8ktSzkYZ+kqOADwFnAacCb01y6ihrkKSejfpKfx0wXVX3VtV3gU8AG0ZcgyR1a9mIz7cS2DPQ3gu8cnBCks3A5tb8VpKvjKi2HiwHvj7uIg4ll467Ao2B/zcX148cbGDUoX9IVbUV2DruOpaiJFNVtXbcdUhz+X9zdEa9vLMPWD3QXtX6JEkjMOrQvwVYk+TkJEcDbwF2jLgGSerWSJd3quqJJO8APgscBVxZVXeNsobOuWymZyv/b45IqmrcNUiSRsRP5EpSRwx9SeqIoS9JHTH0O5DkhCQnjLsOSeNn6C9RSV6a5BNJZoBdwBeS7G99k+OtTtK4GPpL19XAPwE/XFVrquoU4CTgn5n9ziNp7JKsSHJ6e6wYdz098JbNJSrJ7qpac7hj0igkeQXwt8CxPPmp/FXAo8DvVNVt46ptqTP0l6gknwAeBrbz5JfcrQY2Asur6txx1SYluQP4zaraNaf/DODvqurl46ls6TP0l6j2NRebmP3q6pWtey/wL8C2qvqfcdUmHeI30em2HKkjwNCXNHJJLgd+DLiKp/4meh5wX1W9Y1y1LXWGfoeSvLGqrht3HepbkrN46m+i+4AdVXX9+Kpa+gz9DiV5f1VdPO46JI2eob+EJXkZ819J3TO+qqSnl2Rz+2NKOgK8T3+JSnIRs/fjB/hCewT4eJIt46xNOoSMu4ClzCv9JSrJfwI/WVWPz+k/GrjL+/T1bJXk/Kr6yLjrWKq80l+6vge8ZJ7+k9qY9Gz1/nEXsJQ96/4wuhbNu4GdSXbz5C1xLwVOAbwdTmOV5M6DDQF+HcMR5PLOEpbkecA6nvpG7i1V9b/jq0qCJA8BrwcemTsEfL6q5vstVYvAK/0lrKq+B9w87jqkeVwHHFNVd8wdSHLT6Mvph1f6ktQR38iVpI4Y+pLUEUNfXUjyrfY8maSSvHNg7G+SvH2g/Z4kX05yR5JbkpzX+m9KsnbOcb+aZPlA+zVJrmvbb08yk+T2JLuTfDbJqwbmfjTJfe08dyT5/BH7B5AaQ1892g+8q31Q7SmS/BbwC8C6qnoFsJ7hPiF6dVWd1j4MdwnwqSQ/MTD+3qp6RXu86iDHkBaNoa8ezQA7mf2DMnO9D/jtqvomQFV9s6q2L8ZJq+pGYCuweTGOJy2Eoa9eXQq8J8lRBzqSvBj4oaq69wie9zbgZQPtDwws73zsCJ5XArxPX52qqnuT7AJ+ddhDPcO+A+YuFb23qq4ZsgbpGfNKXz37M+AiWhC3JZ1vJfnRwzjGN4DjB9onAF9/mvmnAX61tcbG0Fe3qurLwN3ALw50/znwobbUQ5JjDty9cxA3AW9rc48Cfh24cb6JSX6O2fX8vx+6eGmBXN5R7/4UuH2gfQVwDHBLkseBx4G/HBj/dOsH+HfgN4ArkvwHs78xfAb4h4H5v5LkZ4AXAvcBvzznj9h8IMkfDbTXVdV3F+Hnkubl1zBIUkdc3pGkjhj6ktQRQ1+SOmLoS1JHDH1J6oihL0kdMfQlqSP/B0OrqRbIC4XcAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "seen_df.groupby('INCLUDE').size().plot.bar()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "\n",
    "# Here we define the whole pipeline starting from creating features to training a classifier\n",
    "\n",
    "pipeline = Pipeline([\n",
    "    ('vect', TfidfVectorizer()),\n",
    "    ('clf', SVC(probability=True,  class_weight=\"balanced\")),\n",
    "])\n",
    "\n",
    "# Now we define the parameter space we want to search. This gets computationally expensive the \n",
    "# more parameters we search, as we fit a model for every unique combination of parameters (there are fancier and smarter ways to do this)\n",
    "\n",
    "parameters = [\n",
    "    {\n",
    "        'vect__max_df': (0.5,),\n",
    "        'vect__min_df': (5, 10, 15),\n",
    "        'vect__ngram_range': ((1, 1), (1, 2)),  \n",
    "        'clf__kernel': ['rbf'], \n",
    "        'clf__gamma': [1e-3, 1e-4], # Because only the rbf kernel has the gamma parameters, we do this separately\n",
    "        'clf__C': [1, 1e1, 1e2]\n",
    "    },\n",
    "    {\n",
    "        'vect__max_df': (0.5,),\n",
    "        'vect__min_df': (5, 10, 15),\n",
    "        'vect__ngram_range': ((1, 1), (1, 2)),  \n",
    "        'clf__kernel': ['linear'], \n",
    "        'clf__C': [1e2, 1e3]\n",
    "    }\n",
    "]\n",
    "\n",
    "clf = GridSearchCV(pipeline, parameters, scoring=\"f1\", n_jobs=4, verbose=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "retrain = False\n",
    "if retrain:\n",
    "    # Now we fit them, by default we try a model for each set of parameters for 5 folds of train/test data\n",
    "    # Even with this running in parallel (n_jobs=8) this might be time to get a coffee\n",
    "    clf.fit(df.loc[seen_index, 'content'], df.loc[seen_index,'INCLUDE'])\n",
    "    # The best model is the combination of parameters with the highest mean score across 5 folds\n",
    "    clf.best_estimator_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "if retrain:\n",
    "    # We can also put the results into a dataframe\n",
    "    results = pd.DataFrame(clf.cv_results_)\n",
    "    results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "if retrain:\n",
    "    # Now if we want to look at what the individual parameters do we can plot the range of results obtained\n",
    "    # with each value of the parameter\n",
    "    def param_boxes(results, param, scorer=\"F1\"):\n",
    "        fig, ax = plt.subplots(figsize=(7,4))\n",
    "        # A list of values for our boxplot\n",
    "        x = []\n",
    "        labels = []\n",
    "        for i, (name, group) in enumerate(results.groupby(param)):\n",
    "            labels.append(name)\n",
    "            x.append(group['mean_test_score'])\n",
    "        ax.boxplot(x)\n",
    "        ax.set_xticklabels(labels)\n",
    "        ax.set_xlabel(param)\n",
    "        ax.set_ylabel(scorer)\n",
    "        plt.show()\n",
    "\n",
    "    param_boxes(results, \"param_clf__C\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "if retrain:\n",
    "    # This also works for a combination of params\n",
    "    param_boxes(results, [\"param_clf__kernel\",\"param_clf__C\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "if retrain:\n",
    "    param_boxes(results, \"param_vect__ngram_range\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "if retrain:\n",
    "    param_boxes(results, \"param_vect__min_df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "if retrain:\n",
    "    param_boxes(results, \"param_vect__max_df\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "if retrain:\n",
    "    clf = clf\n",
    "else:\n",
    "    clf = Pipeline(steps=[('vect', TfidfVectorizer(max_df=0.5, min_df=5)),\n",
    "                ('clf',\n",
    "                 SVC(C=100.0, class_weight='balanced', gamma=0.001,\n",
    "                     probability=True))])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# We can fit our best model with the whole training set now \n",
    "clf.fit(df.loc[seen_index, 'content'], df.loc[seen_index,'INCLUDE'])\n",
    "# And obtain predictions for all that we have not yet seen\n",
    "y_pred = clf.predict_proba(df.loc[unseen_index,'content'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/max/software/django-tmv/tmv/venv/lib/python3.6/site-packages/ipykernel_launcher.py:2: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  \n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "45708\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZhcdZ3v8fe3q/f0lnR3tk5Cd1YIW4AmZGGTzYAI1w1BVFzxeq+OI96ZwcsM43Xucx31Ga8bKnjHYUYRjIpOjJEMm2EgCSQhC1noTmfvLL1U0nt6qerf/aNOh0qnl0pS1bV9Xs9TT536nVN1vr/kPJ+c/M5mzjlERCT5ZcS7ABERiQ4FuohIilCgi4ikCAW6iEiKUKCLiKSIzHituKyszFVWVsZr9SIiSWnTpk3NzrnyoebFLdArKyvZuHFjvFYvIpKUzOzAcPM05CIikiIU6CIiKUKBLiKSIhToIiIpQoEuIpIiRg10M/uZmTWa2fZh5puZfd/M6sxsm5ldGf0yRURkNJHsoT8JLBth/u3AHO/1IPDj8y9LRETO1qiB7px7BTg+wiJ3A//mQtYDJWY2JVoFioikip5AkH/809tsOdQSk9+Pxhh6BXAo7HO913YGM3vQzDaa2campqYorFpEJHk0tvXwkzV7qD3WHpPfH9ODos65J5xz1c656vLyIa9cFRFJWcfaugGYXJwbk9+PRqAfBqaHfZ7mtYmISJgjLScBmJLAgb4C+Lh3tssioNU5dzQKvysiklJqjrXjyzBmlObH5PdHvTmXmT0N3AiUmVk98PdAFoBz7ifAKuAOoA7oAj4Zk0pFRJLc5oMtXDSlkJxMX0x+f9RAd87dN8p8B/z3qFUkIpKCegP9bDnUwoevnj76wudIV4qKiIyBbfUtnOwLsmjmhJitQ4EuIjIGXt8XupznmqrSmK1DgS4iMgbW7/Uzb1Ih48dlx2wdCnQRkRjrC/az6cAJronhcAso0EVEYm5bfQtdvUEWzYzdcAso0EVEYm5tnR9AgS4ikuxe29PMRVOKmBDD8XNQoIuIxFRHT4BNB05w/ZyymK9LgS4iEkPr9/jpCzpumBv7GxIq0EVEYujVumZyszK4qnJ8zNelQBcRiaFX65pZWFUas/u3hFOgi4jEyNHWk9Q1dnDd7NiPn4MCXUQkZl7d3QzAtWNwQBQU6CIiMfNaXTNlBTlcOLlwTNanQBcRiQHnHK/t8bNkVilmNibrVKCLiMRAXWMHTe09LJkV26tDwynQRURiYO2e0OX+S8fogCgo0EVEYmLtnmYqSvKYPiE2zw8digJdRCTKgv2O9XuPs3T22A23gAJdRCTqttW30Hqyb0yHW0CBLiISda/VeeefK9BFRJLba3V+LpxcSGlBzpiuV4EuIhJF3X1BNh04wXVjdHVoOAW6iEgUbTpwgt5gP0tmKdBFRJLa63v9ZBhUj8HtcgdToIuIRNEb+48zf2oRhblZY75uBbqISJT0BIJsPtjC1ZUT4rJ+BbqISJRsPthCT6CfRTPH9oKiAQp0EZEoea2uGV+GjekNucIp0EVEomTtHj+XVhTHZfwcFOgiIlHR2RNg66EWFsdp7xwiDHQzW2ZmNWZWZ2YPDzF/hpm9bGabzWybmd0R/VJFRBLXxgMnCPQ7Fsdp/BwiCHQz8wGPAbcD84H7zGz+oMX+FljunLsCuBf4UbQLFRFJZOv2+MnyWVzOPx8QyR76QqDOObfXOdcLPAPcPWgZBxR508XAkeiVKCKS+Nbt9XP5tBLyszPjVkMkgV4BHAr7XO+1hfsa8FEzqwdWAV8c6ofM7EEz22hmG5uams6hXBGRxNPe3cf2w61xHT+H6B0UvQ940jk3DbgD+LmZnfHbzrknnHPVzrnq8vLyKK1aRCS+Nuw/TjDO4+cQWaAfBqaHfZ7mtYX7NLAcwDm3DsgFxv7ONCIicbBuj59sXwZXXhC/8XOILNA3AHPMrMrMsgkd9FwxaJmDwM0AZnYRoUDXmIqIpIV1e/1cMaOE3CxfXOsYNdCdcwHgC8BqYBehs1l2mNnXzewub7GvAJ81s63A08AnnHMuVkWLiCSK1q4+dhxpi/v4OUBEh2Odc6sIHewMb3s0bHonsDS6pYmIJL7X9/lxjriPn4OuFBUROS/r9vrJycxgwYySeJeiQBcROR/r9viprhxPTmZ8x89BgS4ics6Od/by9rH2hBhuAQW6iMg5e7WuGYClsxPjLG0FuojIOXp1dxNFuZlcNi3+4+egQBcROSfOOV6r87NkVhm+DIt3OYACXUTknBw83sXhlpMsmZ0Y4+egQBcROSfr9/oB4va4uaEo0EVEzsG6PX7KCnKYVV4Q71JOUaCLiJwl5xzr9vpZNHMCZokxfg4KdBGRs7bf30VDWw+LEuT88wEKdBGRs/Tq7tDNZK9NkPPPByjQRUTO0rq9fqYU53JBaX68SzmNAl1E5CwE+x2v7m7m2tllCTV+Dgp0EZGzsuNIK23dgYS53D+cAl1E5Cz85+7Q/VuunaNAFxFJauv3+pk7qYCygpx4l3IGBbqISIRO9gZ5fd9xrp1dHu9ShqRAFxGJ0Bv7j9Mb6Of6uYk33AIKdBGRiP25ppGczAyuqUqsC4oGKNBFRCL0Sm0Ti2aWkpcd/8fNDUWBLiISgSMtJ9nT1Ml1CXh2ywAFuohIBF71Tle8bk5iHhAFBbqISERefLuBiYU5zJ2UOLfLHUyBLiIyivoTXfzHzgbed0VFwl3uH06BLiIyij/XNOEcfPjq6fEuZUQKdBGRUazd08yU4lyqysbFu5QRKdBFREYQCPbzWp2fpQl4d8XBFOgiIiPYWt9C68k+bpyXuGe3DFCgi4iMYE1NExkG1yXo/VvCRRToZrbMzGrMrM7MHh5mmXvMbKeZ7TCzX0a3TBGR+PhzbRMLppdQnJ8V71JGNWqgm5kPeAy4HZgP3Gdm8wctMwf4KrDUOXcx8JcxqFVEZEw1d/Swrb6VG+dNjHcpEYlkD30hUOec2+uc6wWeAe4etMxngceccycAnHON0S1TRGTsDVwdmgzj5xBZoFcAh8I+13tt4eYCc83sNTNbb2bLhvohM3vQzDaa2campqZzq1hEZIysqW2idFw2l0wtjncpEYnWQdFMYA5wI3Af8FMzKxm8kHPuCedctXOuurw8Of7FE5H01N/veKW2ievnlpORkdinKw6IJNAPA+GXR03z2sLVAyucc33OuX1ALaGAFxFJSjuPtuHv7E3ouysOFkmgbwDmmFmVmWUD9wIrBi3ze0J755hZGaEhmL1RrFNEZEytqQ0NCyfiw6CHM2qgO+cCwBeA1cAuYLlzboeZfd3M7vIWWw34zWwn8DLwV845f6yKFhGJtRd2NXDZtGImFubGu5SIZUaykHNuFbBqUNujYdMOeMh7iYgkNX9HD1sOtfClm5Nr5FhXioqIDPLS2404B7dcNCnepZwVBbqIyCAv7mpkSnEuF08tincpZ0WBLiISprsvyCu7m7jpwokJf3fFwRToIiJh1u/109UbTLrhFlCgi4icZvWOBvKzfSyeVRrvUs6aAl1ExNPf73h+ZwM3XTiR3CxfvMs5awp0ERHP5kMtNHf0cOv85BtuAQW6iMgpf3rrKNm+DN51YXLcLncwBbqICOCc4/ldDSydXUpRbuI/zGIoCnQREaCmoZ0D/i5uSdLhFlCgi4gAsHLrUTIMbps/Od6lnDMFuoikPeccq946yuJZpZQX5sS7nHOmQBeRtFfT0M7e5k7ec+nUeJdyXiK622Is+P1+nnzyydPaLr74Yq6++mr6+vp46qmnzvjOggULWLBgAV1dXSxfvvyM+dXV1VxyySW0trbyu9/97oz5ixcvZt68eTQ3N7Ny5coz5l9//fXMnDmTY8eO8dxzz50x/+abb2b69OkcOnSIF1988Yz5y5YtY/Lkyezdu5dXXnnljPl33nknZWVl1NTUsG7dujPmv+9976O4uJjt27ezcePGM+bfc8895Ofns2XLFrZs2XLG/Pvvv5+srCw2bNjAjh07zpj/iU98AoC1a9dSW1t72rysrCzuv/9+ANasWcO+fftOm5+fn88999wDwAsvvEB9ff1p84uKinj/+98PwHPPPcexY8dOm19aWsp73/teAP7whz/g959+d+XJkyezbFnoyYXPPvssbW1tp82fNm0at9xyCwDLly+nq6vrtPlVVVXccMMNADz11FP09fWdNn/u3LksWbIE4IztDrTtpfu2t/J46KyW3rp1PLnrpdPmJ/q2F0576CKS1vqdY+W2owDk58RtHzcqLHQr87FXXV3thtoTEBEZS5sOnOADP17L9+5dwN0LKuJdzqjMbJNzrnqoedpDF5G09oetR8jOzOCmJL2YKJwCXUTSViDYz8ptR7lp3kQKk/RionAKdBFJW2/sO05zRw93LUjus1sGKNBFJG39YdtR8rN9vGte8g+3gAJdRNJUd1+QlVuPsOziyeRlJ9+tcoeiQBeRtLR6xzHaewJ8sHpavEuJGgW6iKSlX2+sp6Ikj0VVyfdkouEo0EUk7Rz0d/FqXTP3Xj2djIzkehD0SBToIpJ2fvtmPWbwgatSZ7gFFOgikmb6+x2/2VTP0lllTC3Ji3c5UaVAF5G08tqeZg63nOSeq6fHu5SoU6CLSFpZvrGe4rwsbkviJxMNR4EuImnjeGcvq7cf431XVJCblRrnnodToItI2nj6jYP0Bvv5yDUz4l1KTCjQRSQt9AX7+fm6A1w7u4y5kwrjXU5MRBToZrbMzGrMrM7MHh5huQ+YmTOzIe/VKyISL89tP8axtm4+ubQy3qXEzKiBbmY+4DHgdmA+cJ+ZzR9iuULgS8Dr0S5SROR8/ctr+7igND9lbsQ1lEj20BcCdc65vc65XuAZ4O4hlvsH4JtAdxTrExE5b1sPtfDmwRYeWFyZUleGDhZJoFcAh8I+13ttp5jZlcB059wfR/ohM3vQzDaa2campqazLlZE5Fw8uXY/47J9fCiFbsQ1lPM+KGpmGcB3gK+Mtqxz7gnnXLVzrrq8vPx8Vy0iMqrG9m5WbjvCh6qnp8RTiUYSSaAfBsIvqZrmtQ0oBC4B/mxm+4FFwAodGBWRRPDU+oME+h0PLKmMdykxF0mgbwDmmFmVmWUD9wIrBmY651qdc2XOuUrnXCWwHrjLObcxJhWLiESouy/IL9Yf4F3zJlJVNi7e5cTcqIHunAsAXwBWA7uA5c65HWb2dTO7K9YFioicq99vPoy/s5fPXFcV71LGRGYkCznnVgGrBrU9OsyyN55/WSIi5ycQ7OfxV/Zy8dQiFs9MnYdYjERXiopISvrjW0fZ19zJF2+ajVnqnqoYToEuIimnv9/xg5fqmDupgNvmT453OWNGgS4iKWfV9qPUNXbwxZvmpPSFRIMp0EUkpQSC/fzf52uZM7GAOy6dEu9yxpQCXURSyrObD7OnqZOv3DYXXxrtnYMCXURSSHdfkO8+X8vl00t498XpM3Y+QIEuIinjX9fu50hrN3+zbF7anNkSToEuIimhqb2HH75Ux7vmlbNkVlm8y4kLBbqIpIRXapto7wnwldvmxbuUuFGgi0jSc86xfOMhxudnpezj5SKhQBeRpLd6RwOv7zvOQ7fOJTszfWMtfXsuIimhJxDkG3/axZyJBdy3cEa8y4krBbqIJLV/XbufA/4u/u7O+WT60jvS0rv3IpLUmjt6+MGLoTNbrp+rp6Ap0EUkaX3n+Vq6+oI88p758S4lISjQRSQpvVXfyjNvHORjiy5g9sSCeJeTEBToIpJ0gv2OR37/FhPG5fDQbXPjXU7CUKCLSNJ56vUDbKtv5e/uvIii3Kx4l5MwFOgiklSOtp7kW8/VcO3sMu66fGq8y0koCnQRSRrOOR753XaC/Y7/875L0/IGXCNRoItI0vjtm4d56e1GvnLbXGaU5se7nISjQBeRpHDoeBdfW7GDhZUT+OTSqniXk5AU6CKS8IL9joeWbwHgn+65PO2eRBSpzHgXICIymh+9XMeG/Sf4pw9dzvQJGmoZjvbQRSShralt4jsv1HL3gqm8/8qKeJeT0BToIpKwjrSc5C+f2cy8SYV84/06q2U0CnQRSUhdvQE+/4tN9Ab6+dH9V5KfrRHi0ehPSEQSTiDYz188vZm3Drfy+MeqmVmue7VEQoEuIgnFOcfX/rCDF3Y18g93X8yt8yfFu6SkoSEXEUkoP16zh1+sP8jnbpjJxxZXxrucpBJRoJvZMjOrMbM6M3t4iPkPmdlOM9tmZi+a2QXRL1VEUt2/bznMt56r4a7Lp/I3774w3uUknVED3cx8wGPA7cB84D4zG3w3+c1AtXPuMuA3wLeiXaiIpLa1e5r5H7/eyqKZE/j2hy4jQxcPnbVI9tAXAnXOub3OuV7gGeDu8AWccy8757q8j+uBadEtU0RSWc2xdj73801Ulo7j8Y9Vk5Ppi3dJSSmSQK8ADoV9rvfahvNp4E/nU5SIpI+dR9r4yE/Xk5fl48lPLaQ4T/c3P1dRPcvFzD4KVAM3DDP/QeBBgBkzZkRz1SKShLYeauET//IGuVk+nvrMNVSU5MW7pKQWyR76YWB62OdpXttpzOwW4BHgLudcz1A/5Jx7wjlX7ZyrLi/XE7pF0tmru5u576frGZeTydOfXaRzzaMgkkDfAMwxsyozywbuBVaEL2BmVwCPEwrzxuiXKSKpZOW2I3zyyTeYMSGfZz+/hMqycfEuKSWMGujOuQDwBWA1sAtY7pzbYWZfN7O7vMW+DRQAvzazLWa2YpifE5E09/P1B/ji05tZML2EX31uMROLcuNdUsqIaAzdObcKWDWo7dGw6VuiXJeIpBjnHN97cTfffWE3t1w0kR9+5Epys3Q2SzTp0n8RibnuviD/89m3eHbzYT5w5TS++YFLyfTpQvVoU6CLSEzVNXbwhV++ydvH2vnyLXP5i5tn6za4MaJAF5GYefbNev7299vJzfLx5Cev5sZ5E+NdUkpToItI1HX2BPjaih38elM9C6sm8P17r2BysQ5+xpoCXUSiatOB43z5V1s5dKKLL940my/dPEfj5WNEgS4iUdHS1cs3n6vhmQ0HqSjJY/nnFnN15YR4l5VWFOgicl6cc/xmUz3f+NPbtJ7s41NLq/jyrXMpyFG8jDX9iYvIOXvz4Am+sWoXG/af4MoZJfzv/3Ip86cWxbustKVAF5GzVtfYzrdX17B6RwNlBdl88wOX8qGrpuse5nGmQBeRiO1r7uQHL+3m95sPk5+dyZdvmctnrqtinIZXEoL+FkRkVDuOtPKTNXv547YjZPky+PS1VfzXG2ZRWpAT79IkjAJdRIbU3+9YU9vEz17bx3/ubmZcto/PXj+TT19bxcRCnVOeiBToInIaf0cPz755mF+8foAD/i4mFeXw18vmcf81F+hpQglOgS4idPcFefntRp7dfJiX324k0O+4unI8D906lzsunUKWLgxKCgp0kTTV3RfkldomVr11lBd3NdLeE6C8MIdPXVvFB6+axtxJhfEuUc6SAl0kjTS2d/NKbTMvvd3AmpomOnuDlORncfulk7nzsqksmVWqy/STmAJdJIV19wV588AJXq1rZk1tEzuOtAFQXpjD3VdU8O6LJ7NkVqmGVFKEAl0khXT0BNh04ARv7POzYd8JthxqoTfYjy/DuOqC8fzVu+dxw9xy5k8p0kVAKUiBLpKk+oL97G7o4K3DLWw51MqWQy3UNrQT7Hf4MoxLKor5xNJKrqmawMKqCRTm6gyVVKdAF0kC7d191DZ0sPNoGzuPtLHzSCu7jrXTG+gHoCg3k8unl3DLRbNYWDWBK2eM19WbaUh/4yIJpPVkH/uaO6lr7GB3Qzs1De3sbujgcMvJU8sU52Uxf0oRDyy+gEsqirm0opjK0nEaQhEFukisdfYEaGzvobGtm4b2HvwdPbSe7Dv1amrv4VhrN8faumnvDpz6XrYvg5nl46iuHM9HJs1g7qRCLppSSEVJnp7JKUNSoIuch75gP41eIB9pOcnhlpMc8V71J0Kfw0M6XGFOJkV5WZQVZDOzfBxLZpUytSSPmeUFVJWNo7I0X6cQyllRoIsMoScQxN/RS1N7D43tPTR5r8b2bhraumlo6+FYWzfNHT04d/p3i3IzmVqSx9SSPBZWTWBKcR4TC3OYVJTLxKIcSsdlU5yXpbCWqFOgS1roCQRp6erjeGfvaS9/Zy/+jh78Hb34O3vwd/bS3N5D2zB71SX5WUwuymVSUS7zpxQxqTiXKcW5TC7OpaIkjynFuTqbROJGgS5JI9jv6OgO0NYdGntu6+6j7WTAe++jpauPlpO9ofeuPk509Z567+oNDvmbZjA+P5vScdmUFmRz0ZQiSmdnU1aQQ1lBDuWFOUwsDL2XFeSQnam9aklcCnQZE72Bfjp7AnT0BOjsDdDRHZpu9947ugO0d/fR1h2gbVBYt3tt7T1D7zUP8GUYJXlZFOdnUZIX2pO+cHIRJflZjM/PotgL7vH5ofAen5/N+HwNfUjqUKDLaZxz9AT66eoN0tkTCL33BujqCb139gTo7A3SNfh9YF6PNz1oXl/Qjbpus3cOFBblZlGUl8n0CfmnpkPvWRTlZlIY3pYbCvHCnEyduidpTYGehJxz9AUdJ3uDdPUFQu+9Qbr7Qu/h0yf7gpzsDZxqD31nIGwDpwe39x7oHz18B+Rn+8jPzmRcTui9IMdHSX42FeO99mwf+Tmh93E5mYzLzqQgN5OCnEzG5WRSmDvwyiI/y6dAFjkPCvQoG9jD7e4bCNPQ+6mA7T29/WRfkO6w6YEwfmd+/2nzB9qDZxG6AJkZRl62j7wsH/lh4TphXDbTx+efantn3kAQZ5Kf4wu9Z/soyHnnc54CWCShpG2g9/c7uvqCp8Z1O7oDp6YH9ma7et8ZcjjZe3ogd3ltQ4XwWWYtENrTzcvykZvlOxW8edk+SvKyyCvKJTcrgzwvVEPLDf7s8/aWfWHBnXlqWgfzRFJfRIFuZsuA7wE+4P855/5x0Pwc4N+AqwA/8GHn3P7oljq0vmA/DW3d+Dt6Od7Vy/GOd05Ha+nqPeOg2sA4b2dv4Izzh4eT7csgLyws87N95GdlUpyfzZSBAM72ke9ND4Rr3qDPuVnvtOWFTedkZujKPxE5b6MGupn5gMeAW4F6YIOZrXDO7Qxb7NPACefcbDO7F/gm8OFYFOyc49k3D/P8zgZqG9rZ7+8cco84y2eU5Icu4CjMDYXvtAn5FGSHxm4LckJDC6fGc0+1hw0peOGte0WLSDKIZA99IVDnnNsLYGbPAHcD4YF+N/A1b/o3wA/NzJyLdB84ciu3HeUrv95KRUkel00r5j2XTaGiJI/SghwmjAudljahIJvCnEzt9YpIWokk0CuAQ2Gf64FrhlvGORcws1agFGgOX8jMHgQeBJgxY8Y5FZyb5ePW+ZN4/KNX6YCciEiYMT0o6px7AngCoLq6+pz23m+dP4lb50+Kal0iIqkgksHhw8D0sM/TvLYhlzGzTKCY0MFREREZI5EE+gZgjplVmVk2cC+wYtAyK4AHvOkPAi/FYvxcRESGN+qQizcm/gVgNaHTFn/mnNthZl8HNjrnVgD/DPzczOqA44RCX0RExlBEY+jOuVXAqkFtj4ZNdwMfim5pIiJyNnSCtYhIilCgi4ikCAW6iEiKUKCLiKQIi9fZhWbWBBw4x6+XMegq1BSVDv1Mhz5CevQzHfoI8e/nBc658qFmxC3Qz4eZbXTOVce7jlhLh36mQx8hPfqZDn2ExO6nhlxERFKEAl1EJEUka6A/Ee8Cxkg69DMd+gjp0c906CMkcD+TcgxdRETOlKx76CIiMogCXUQkRSRdoJvZMjOrMbM6M3s43vUMxcx+ZmaNZrY9rG2CmT1vZru99/Feu5nZ973+bDOzK8O+84C3/G4zeyCs/Soze8v7zvfNe9becOuIUR+nm9nLZrbTzHaY2ZdStJ+5ZvaGmW31+vm/vPYqM3vdq+1X3q2lMbMc73OdN78y7Le+6rXXmNm7w9qH3KaHW0cM++ozs81mtjKF+7jf26a2mNlGry11tlnnXNK8CN2+dw8wE8gGtgLz413XEHVeD1wJbA9r+xbwsDf9MPBNb/oO4E+AAYuA1732CcBe7328Nz3em/eGt6x53719pHXEqI9TgCu96UKgFpifgv00oMCbzgJe92paDtzrtf8E+Lw3/d+An3jT9wK/8qbne9trDlDlbce+kbbp4dYRw74+BPwSWDnS+pO8j/uBskFtKbPNxuwPLkZ/GYuB1WGfvwp8Nd51DVNrJacHeg0wxZueAtR4048D9w1eDrgPeDys/XGvbQrwdlj7qeWGW8cY9fffgVtTuZ9APvAmoWfqNgOZg7dLQs8NWOxNZ3rL2eBtdWC54bZp7ztDriNGfZsGvAjcBKwcaf3J2kdvHfs5M9BTZptNtiGXoR5YXRGnWs7WJOfcUW/6GDDwYNTh+jRSe/0Q7SOtI6a8/3JfQWjvNeX66Q1FbAEagecJ7W22OOcCQ9R22gPTgYEHpp9t/0tHWEcsfBf4a6Df+zzS+pO1jwAO+A8z22Shh9ZDCm2zY/qQaAlxzjkzi+n5omOxDgAzKwB+C/ylc67NGzIcsxrGaB1BYIGZlQC/Ay6M5frGmpndCTQ65zaZ2Y3xrifGrnXOHTazicDzZvZ2+Mxk32aTbQ89kgdWJ6oGM5sC4L03eu3D9Wmk9mlDtI+0jpgwsyxCYf6Uc+7ZUWpI2n4OcM61AC8TGhoosdAD0QfXNtwD08+2//4R1hFtS4G7zGw/8AyhYZfvjbD+ZOwjAM65w957I6F/nBeSQttssgV6JA+sTlThD9J+gNCY80D7x70j6ouAVu+/ZquB28xsvHdE/DZC44tHgTYzW+QdQf/4oN8aah1R5637n4FdzrnvhM1KtX6We3vmmFkeoeMEuwgF+weH6edQD0xfAdzrnSFSBcwhdABtyG3a+85w64gq59xXnXPTnHOV3vpfcs7dn0p9BDCzcWZWODBNaFvbTipts7E8ABGLF6Ejz7WExjEfiXc9w9T4NHAU6CM0jvZpQuOFLwK7gReACd6yBjzm9ectoDrsdz4F1HmvT4a1VxPaEPcAP+SdK09wPDEAAACKSURBVH6HXEeM+ngtofHIbcAW73VHCvbzMmCz18/twKNe+0xCYVUH/BrI8dpzvc913vyZYb/1iNeXGryzH0bapodbR4y33Rt55yyXlOqjt66t3mvHQB2ptM3q0n8RkRSRbEMuIiIyDAW6iEiKUKCLiKQIBbqISIpQoIuIpAgFuohIilCgi4ikiP8PbeCksW/gAFQAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "df['prediction'] = None\n",
    "df['prediction'][unseen_index] = y_pred[:,1]\n",
    "y_pred.shape\n",
    "\n",
    "plt.close()\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "\n",
    "pdf = df.sort_values('prediction').reset_index(drop=True)\n",
    "ax.plot(pdf.index,pdf['prediction'])\n",
    "ax.axhline(0.5,linestyle=\"--\",color=\"grey\")\n",
    "\n",
    "print(pdf[pdf['prediction']>0.5].shape[0])\n",
    "\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2240"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "django.db.connection.close()\n",
    "assigned = set(DocOwnership.objects.filter(query=8843).values_list('doc_id',flat=True))\n",
    "len(assigned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "django.db.connection.close()\n",
    "pairs = [\n",
    "    (['vasudevan@mcc-berlin.net','lucy.banisch@icloud.com'], 100),      \n",
    "    (['doebbeling@mcc-berlin.net','galm'], 20),   \n",
    "]\n",
    "\n",
    "new_q = Query.objects.get(pk=8843)\n",
    "\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"Probably relevant random sample 16\",\n",
    "    query=new_q,\n",
    ")\n",
    "\n",
    "gov_docs = set(df[df['prediction']>=0.5].id) - assigned\n",
    "\n",
    "if created:\n",
    "\n",
    "    T = Doc.tag.through\n",
    "\n",
    "    for p in pairs:\n",
    "        gov_sample = set(random.sample(gov_docs, p[1]))\n",
    "        gov_docs -= gov_sample\n",
    "\n",
    "        for did in gov_sample:\n",
    "            for u in p[0]:\n",
    "                user = User.objects.get(username=u)\n",
    "                do, created = DocOwnership.objects.get_or_create(\n",
    "                    query=new_q,\n",
    "                    tag=tag,\n",
    "                    doc_id=did,\n",
    "                    user=user\n",
    "                )\n",
    "\n",
    "        dts = [T(doc_id=did, tag=tag) for did in gov_sample]\n",
    "        T.objects.bulk_create(dts)\n",
    "\n",
    "    tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "django.db.connection.close()\n",
    "pairs = [\n",
    "    (['vasudevan@mcc-berlin.net','lucy.banisch@icloud.com'], 80),      \n",
    "    (['doebbeling@mcc-berlin.net','galm'], 20),   \n",
    "]\n",
    "\n",
    "new_q = Query.objects.get(pk=8843)\n",
    "\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"Probably relevant random sample 15\",\n",
    "    query=new_q,\n",
    ")\n",
    "\n",
    "gov_docs = set(df[df['prediction']>=0.5].id) - assigned\n",
    "\n",
    "if created:\n",
    "\n",
    "    T = Doc.tag.through\n",
    "\n",
    "    for p in pairs:\n",
    "        gov_sample = set(random.sample(gov_docs, p[1]))\n",
    "        gov_docs -= gov_sample\n",
    "\n",
    "        for did in gov_sample:\n",
    "            for u in p[0]:\n",
    "                user = User.objects.get(username=u)\n",
    "                do, created = DocOwnership.objects.get_or_create(\n",
    "                    query=new_q,\n",
    "                    tag=tag,\n",
    "                    doc_id=did,\n",
    "                    user=user\n",
    "                )\n",
    "\n",
    "        dts = [T(doc_id=did, tag=tag) for did in gov_sample]\n",
    "        T.objects.bulk_create(dts)\n",
    "\n",
    "    tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "django.db.connection.close()\n",
    "pairs = [\n",
    "    (['vasudevan@mcc-berlin.net','lucy.banisch@icloud.com'], 60),      \n",
    "    (['ceposadap@gmail.com','edmondson@hertie-school.org'], 40),   \n",
    "    (['vasudevan@mcc-berlin.net','ceposadap@gmail.com'], 40)\n",
    "]\n",
    "\n",
    "new_q = Query.objects.get(pk=8843)\n",
    "\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"Probably relevant random sample 12\",\n",
    "    query=new_q,\n",
    ")\n",
    "\n",
    "gov_docs = set(df[df['prediction']>=0.5].id)\n",
    "\n",
    "if created:\n",
    "\n",
    "    T = Doc.tag.through\n",
    "\n",
    "    for p in pairs:\n",
    "        gov_sample = set(random.sample(gov_docs, p[1]))\n",
    "        gov_docs -= gov_sample\n",
    "\n",
    "        for did in gov_sample:\n",
    "            for u in p[0]:\n",
    "                user = User.objects.get(username=u)\n",
    "                do, created = DocOwnership.objects.get_or_create(\n",
    "                    query=new_q,\n",
    "                    tag=tag,\n",
    "                    doc_id=did,\n",
    "                    user=user\n",
    "                )\n",
    "\n",
    "        dts = [T(doc_id=did, tag=tag) for did in gov_sample]\n",
    "        T.objects.bulk_create(dts)\n",
    "\n",
    "    tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>title</th>\n",
       "      <th>user</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Factors influencing German house owners' prefe...</td>\n",
       "      <td>lucy.banisch@icloud.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Increased energy efficiency and the rebound ef...</td>\n",
       "      <td>lucy.banisch@icloud.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Socially responsible or reprehensible? Investo...</td>\n",
       "      <td>lucy.banisch@icloud.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Carbon Prices for the Next Hundred Years</td>\n",
       "      <td>lucy.banisch@icloud.com</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>Determinants of carbon accounting disclosure: ...</td>\n",
       "      <td>lucy.banisch@icloud.com</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                               title                     user\n",
       "0  Factors influencing German house owners' prefe...  lucy.banisch@icloud.com\n",
       "1  Increased energy efficiency and the rebound ef...  lucy.banisch@icloud.com\n",
       "2  Socially responsible or reprehensible? Investo...  lucy.banisch@icloud.com\n",
       "3           Carbon Prices for the Next Hundred Years  lucy.banisch@icloud.com\n",
       "4  Determinants of carbon accounting disclosure: ...  lucy.banisch@icloud.com"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "django.db.connection.close()\n",
    "pairs = []\n",
    "for do in DocOwnership.objects.filter(tag=tag,user__username=\"vasudevan@mcc-berlin.net\"):\n",
    "    odo = DocOwnership.objects.filter(tag=tag, doc=do.doc).exclude(pk=do.id).first()\n",
    "    pairs.append({\"title\":odo.doc.title,\"user\":odo.user})\n",
    "df = pd.DataFrame.from_dict(pairs)\n",
    "df.to_csv('document_pairs.csv',index=False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = [\n",
    "    (['vasudevan@mcc-berlin.net','galm'], 40),      \n",
    "    (['ceposadap@gmail.com','edmondson@hertie-school.org'], 20),   \n",
    "    (['lucy.banisch@icloud.com','ceposadap@gmail.com'], 40)\n",
    "]\n",
    "\n",
    "new_q = Query.objects.get(pk=8843)\n",
    "\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"Probably relevant random sample 10\",\n",
    "    query=new_q,\n",
    ")\n",
    "\n",
    "gov_docs = set(df[df['prediction']>=0.5].id)\n",
    "\n",
    "if created:\n",
    "\n",
    "    T = Doc.tag.through\n",
    "\n",
    "    for p in pairs:\n",
    "        gov_sample = set(random.sample(gov_docs, p[1]))\n",
    "        gov_docs -= gov_sample\n",
    "\n",
    "        for did in gov_sample:\n",
    "            for u in p[0]:\n",
    "                user = User.objects.get(username=u)\n",
    "                do, created = DocOwnership.objects.get_or_create(\n",
    "                    query=new_q,\n",
    "                    tag=tag,\n",
    "                    doc_id=did,\n",
    "                    user=user\n",
    "                )\n",
    "\n",
    "        dts = [T(doc_id=did, tag=tag) for did in gov_sample]\n",
    "        T.objects.bulk_create(dts)\n",
    "\n",
    "    tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = [\n",
    "    (['vasudevan@mcc-berlin.net','lucy.banisch@icloud.com'], 80),      \n",
    "    (['doebbeling@mcc-berlin.net','edmondson@hertie-school.org'], 20),   \n",
    "    (['galm','ceposadap@gmail.com'], 40)\n",
    "]\n",
    "\n",
    "new_q = Query.objects.get(pk=8843)\n",
    "\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"Probably relevant random sample 11\",\n",
    "    query=new_q,\n",
    ")\n",
    "\n",
    "gov_docs = set(df[df['prediction']>=0.5].id)\n",
    "\n",
    "if created:\n",
    "\n",
    "    T = Doc.tag.through\n",
    "\n",
    "    for p in pairs:\n",
    "        gov_sample = set(random.sample(gov_docs, p[1]))\n",
    "        gov_docs -= gov_sample\n",
    "\n",
    "        for did in gov_sample:\n",
    "            for u in p[0]:\n",
    "                user = User.objects.get(username=u)\n",
    "                do, created = DocOwnership.objects.get_or_create(\n",
    "                    query=new_q,\n",
    "                    tag=tag,\n",
    "                    doc_id=did,\n",
    "                    user=user\n",
    "                )\n",
    "\n",
    "        dts = [T(doc_id=did, tag=tag) for did in gov_sample]\n",
    "        T.objects.bulk_create(dts)\n",
    "\n",
    "    tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "pairs = [\n",
    "    (['vasudevan@mcc-berlin.net','doebbeling@mcc-berlin.net'], 80),      \n",
    "    (['lucy.banisch@icloud.com','ceposadap@gmail.com'], 160),\n",
    "    (['edmondson@hertie-school.org','ceposadap@gmail.com'], 40)\n",
    "]\n",
    "\n",
    "new_q = Query.objects.get(pk=8843)\n",
    "\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"Probably relevant random sample 13\",\n",
    "    query=new_q,\n",
    ")\n",
    "\n",
    "gov_docs = set(df[df['prediction']>=0.5].id)\n",
    "\n",
    "if created:\n",
    "\n",
    "    T = Doc.tag.through\n",
    "\n",
    "    for p in pairs:\n",
    "        gov_sample = set(random.sample(gov_docs, p[1]))\n",
    "        gov_docs -= gov_sample\n",
    "\n",
    "        for did in gov_sample:\n",
    "            for u in p[0]:\n",
    "                user = User.objects.get(username=u)\n",
    "                do, created = DocOwnership.objects.get_or_create(\n",
    "                    query=new_q,\n",
    "                    tag=tag,\n",
    "                    doc_id=did,\n",
    "                    user=user\n",
    "                )\n",
    "\n",
    "        dts = [T(doc_id=did, tag=tag) for did in gov_sample]\n",
    "        T.objects.bulk_create(dts)\n",
    "\n",
    "    tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "django.db.connection.close()\n",
    "\n",
    "pairs = [\n",
    "    (['vasudevan@mcc-berlin.net','doebbeling@mcc-berlin.net'], 20),      \n",
    "    (['lucy.banisch@icloud.com','vasudevan@mcc-berlin.net'], 60),\n",
    "]\n",
    "\n",
    "new_q = Query.objects.get(pk=8843)\n",
    "\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"Probably relevant random sample 14\",\n",
    "    query=new_q,\n",
    ")\n",
    "\n",
    "gov_docs = set(df[df['prediction']>=0.5].id)\n",
    "\n",
    "if created:\n",
    "\n",
    "    T = Doc.tag.through\n",
    "\n",
    "    for p in pairs:\n",
    "        gov_sample = set(random.sample(gov_docs, p[1]))\n",
    "        gov_docs -= gov_sample\n",
    "\n",
    "        for did in gov_sample:\n",
    "            for u in p[0]:\n",
    "                user = User.objects.get(username=u)\n",
    "                do, created = DocOwnership.objects.get_or_create(\n",
    "                    query=new_q,\n",
    "                    tag=tag,\n",
    "                    doc_id=did,\n",
    "                    user=user\n",
    "                )\n",
    "\n",
    "        dts = [T(doc_id=did, tag=tag) for did in gov_sample]\n",
    "        T.objects.bulk_create(dts)\n",
    "\n",
    "    tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "\n",
    "django.db.connection.close()\n",
    "pairs = []\n",
    "for do in DocOwnership.objects.filter(tag=tag,user__username=\"vasudevan@mcc-berlin.net\"):\n",
    "    odo = DocOwnership.objects.filter(tag=tag, doc=do.doc).exclude(pk=do.id).first()\n",
    "    pairs.append({\"title\":odo.doc.title,\"user\":odo.user})\n",
    "df = pd.DataFrame.from_dict(pairs)\n",
    "df.to_csv('document_pairs.csv',index=False)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import datetime\n",
    "now = datetime.datetime.now().strftime(\"%Y-%m-%d %H:%M\")\n",
    "q = Query.objects.get(pk=9046)\n",
    "tag, created = Tag.objects.get_or_create(\n",
    "    title=\"predicted relevant sample {}\".format(now),\n",
    "    query=q\n",
    ")\n",
    "tag.save()\n",
    "\n",
    "\n",
    "users = q.users.all()\n",
    "uc = len(users)\n",
    "\n",
    "new_docs = (df\n",
    "            .sort_values('prediction',ascending=False)\n",
    "            .head(50)\n",
    "            .reset_index(drop=True)\n",
    "           )\n",
    "\n",
    "for i, row in new_docs.iterrows():\n",
    "    d = Doc.objects.get(id=row['id'])\n",
    "    d.tag.add(tag)\n",
    "    u = users[i % uc]\n",
    "    for u in users:\n",
    "        do, created = DocOwnership.objects.get_or_create(\n",
    "            doc=d,\n",
    "            user=u,\n",
    "            query=q,\n",
    "            tag=tag\n",
    "        )\n",
    "        do.save()\n",
    "    \n",
    "tag.update_tag()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<Tag: Probably relevant random sample 14>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "django.db.connection.close()\n",
    "tag = Tag.objects.get(title=\"Probably relevant random sample 13\")\n",
    "tag\n",
    "dos = DocOwnership.objects.filter(tag=tag,relevant=0,user__username__icontains=\"edmondson\")\n",
    "u = User.objects.get(username=\"galm\")\n",
    "dos.update(user=u)\n",
    "dos.count()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "tmv",
   "language": "python",
   "name": "tmv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}