examples/multiclass_classification.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7a548e7a-117f-4807-a40b-b521b4dfa750",
   "metadata": {},
   "source": [
    "## Multiclass Venn-ABERS calibration"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "02b3cbd8-1608-4dce-9731-e5b76904788c",
   "metadata": {},
   "source": [
    "This notebook illustrates the multi-class Venn-ABERS calibration as implemented in:\n",
    "\n",
    "Manokhin, Valery. \"Multi-class probabilistic classification using inductive and cross Venn–Abers predictors.\" In Conformal and Probabilistic Prediction and Applications, pp. 228-240. PMLR, 2017."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3be815cc-c047-4c27-80aa-9f40f6a4059a",
   "metadata": {},
   "source": [
    "As an example we use the Satellite Image dataset described in more detail here: https://www.openml.org/search?type=data&sort=runs&id=182&status=active"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "8ece9a32-3e29-464d-9df6-bb2254886cfd",
   "metadata": {},
   "source": [
    "## Import libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d1bed877-36cb-426d-a86b-143eb8d5319a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from scipy.io import arff\n",
    "from sklearn.metrics import log_loss, brier_score_loss, accuracy_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelBinarizer\n",
    "\n",
    "from sklearn.datasets import make_classification\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from sklearn.calibration import CalibratedClassifierCV\n",
    "\n",
    "from sklearn.isotonic import IsotonicRegression\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.svm import SVC, LinearSVC\n",
    "from sklearn.naive_bayes import GaussianNB\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier\n",
    "from sklearn.neural_network import MLPClassifier\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "\n",
    "from venn_abers import VennAbersCalibrator"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f290ebed-6add-4bfe-972b-2b89be4261c1",
   "metadata": {},
   "source": [
    "## Load dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "bee00548-e660-4b0d-b621-9d76b745c3c7",
   "metadata": {},
   "outputs": [],
   "source": [
    "data = arff.loadarff('data/dataset_186_satimage.arff')\n",
    "df = pd.DataFrame(data[0])\n",
    "df['class'] = ([int(str(i)[2]) for i in df['class'].values])\n",
    "df['class'] = df['class'].astype('int')-1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "418753ec-f6b8-48dd-94e2-2c8a2956aa8b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Aattr</th>\n",
       "      <th>Battr</th>\n",
       "      <th>Cattr</th>\n",
       "      <th>Dattr</th>\n",
       "      <th>Eattr</th>\n",
       "      <th>Fattr</th>\n",
       "      <th>A1attr</th>\n",
       "      <th>B2attr</th>\n",
       "      <th>C3attr</th>\n",
       "      <th>D4attr</th>\n",
       "      <th>...</th>\n",
       "      <th>D22attr</th>\n",
       "      <th>E23attr</th>\n",
       "      <th>F24attr</th>\n",
       "      <th>A25attr</th>\n",
       "      <th>B26attr</th>\n",
       "      <th>C27attr</th>\n",
       "      <th>D28attr</th>\n",
       "      <th>E29attr</th>\n",
       "      <th>F30attr</th>\n",
       "      <th>class</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.117596</td>\n",
       "      <td>1.241362</td>\n",
       "      <td>1.184036</td>\n",
       "      <td>0.815302</td>\n",
       "      <td>-0.158561</td>\n",
       "      <td>1.256483</td>\n",
       "      <td>1.193546</td>\n",
       "      <td>0.818486</td>\n",
       "      <td>-0.141965</td>\n",
       "      <td>0.879481</td>\n",
       "      <td>...</td>\n",
       "      <td>0.807707</td>\n",
       "      <td>-0.069968</td>\n",
       "      <td>1.219160</td>\n",
       "      <td>1.250463</td>\n",
       "      <td>0.597678</td>\n",
       "      <td>-0.054291</td>\n",
       "      <td>1.233342</td>\n",
       "      <td>1.262255</td>\n",
       "      <td>0.603258</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>-1.205362</td>\n",
       "      <td>-1.249654</td>\n",
       "      <td>-0.077532</td>\n",
       "      <td>0.444886</td>\n",
       "      <td>-0.895959</td>\n",
       "      <td>-0.447579</td>\n",
       "      <td>-0.786760</td>\n",
       "      <td>-0.554203</td>\n",
       "      <td>-0.364672</td>\n",
       "      <td>0.092157</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.192752</td>\n",
       "      <td>-0.736996</td>\n",
       "      <td>-0.969292</td>\n",
       "      <td>-0.844805</td>\n",
       "      <td>-0.400030</td>\n",
       "      <td>-0.725852</td>\n",
       "      <td>-0.344432</td>\n",
       "      <td>-0.594534</td>\n",
       "      <td>-0.183967</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.779075</td>\n",
       "      <td>0.148811</td>\n",
       "      <td>0.042617</td>\n",
       "      <td>-0.243030</td>\n",
       "      <td>0.800057</td>\n",
       "      <td>0.164136</td>\n",
       "      <td>0.053370</td>\n",
       "      <td>-0.448612</td>\n",
       "      <td>0.154978</td>\n",
       "      <td>-0.345245</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.877277</td>\n",
       "      <td>0.671174</td>\n",
       "      <td>-0.006373</td>\n",
       "      <td>-0.425752</td>\n",
       "      <td>-0.662584</td>\n",
       "      <td>0.691889</td>\n",
       "      <td>0.356801</td>\n",
       "      <td>-0.175259</td>\n",
       "      <td>-0.236449</td>\n",
       "      <td>6</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1.146564</td>\n",
       "      <td>0.585831</td>\n",
       "      <td>0.342991</td>\n",
       "      <td>0.021553</td>\n",
       "      <td>0.947536</td>\n",
       "      <td>0.601074</td>\n",
       "      <td>0.353416</td>\n",
       "      <td>0.026550</td>\n",
       "      <td>1.788164</td>\n",
       "      <td>1.010702</td>\n",
       "      <td>...</td>\n",
       "      <td>0.281150</td>\n",
       "      <td>1.412317</td>\n",
       "      <td>1.044084</td>\n",
       "      <td>0.532085</td>\n",
       "      <td>0.282612</td>\n",
       "      <td>1.438068</td>\n",
       "      <td>1.058033</td>\n",
       "      <td>0.842981</td>\n",
       "      <td>0.130923</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>-0.764376</td>\n",
       "      <td>-1.162250</td>\n",
       "      <td>-0.137607</td>\n",
       "      <td>0.180303</td>\n",
       "      <td>-0.969698</td>\n",
       "      <td>-1.146681</td>\n",
       "      <td>-0.126658</td>\n",
       "      <td>0.184937</td>\n",
       "      <td>-0.735851</td>\n",
       "      <td>-1.132569</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.192752</td>\n",
       "      <td>-0.885225</td>\n",
       "      <td>-1.231906</td>\n",
       "      <td>-0.784941</td>\n",
       "      <td>-0.347519</td>\n",
       "      <td>-0.875088</td>\n",
       "      <td>-1.220973</td>\n",
       "      <td>-0.774223</td>\n",
       "      <td>-0.551339</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 37 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      Aattr     Battr     Cattr     Dattr     Eattr     Fattr    A1attr  \\\n",
       "0  0.117596  1.241362  1.184036  0.815302 -0.158561  1.256483  1.193546   \n",
       "1 -1.205362 -1.249654 -0.077532  0.444886 -0.895959 -0.447579 -0.786760   \n",
       "2  0.779075  0.148811  0.042617 -0.243030  0.800057  0.164136  0.053370   \n",
       "3  1.146564  0.585831  0.342991  0.021553  0.947536  0.601074  0.353416   \n",
       "4 -0.764376 -1.162250 -0.137607  0.180303 -0.969698 -1.146681 -0.126658   \n",
       "\n",
       "     B2attr    C3attr    D4attr  ...   D22attr   E23attr   F24attr   A25attr  \\\n",
       "0  0.818486 -0.141965  0.879481  ...  0.807707 -0.069968  1.219160  1.250463   \n",
       "1 -0.554203 -0.364672  0.092157  ... -0.192752 -0.736996 -0.969292 -0.844805   \n",
       "2 -0.448612  0.154978 -0.345245  ... -0.877277  0.671174 -0.006373 -0.425752   \n",
       "3  0.026550  1.788164  1.010702  ...  0.281150  1.412317  1.044084  0.532085   \n",
       "4  0.184937 -0.735851 -1.132569  ... -0.192752 -0.885225 -1.231906 -0.784941   \n",
       "\n",
       "    B26attr   C27attr   D28attr   E29attr   F30attr  class  \n",
       "0  0.597678 -0.054291  1.233342  1.262255  0.603258      0  \n",
       "1 -0.400030 -0.725852 -0.344432 -0.594534 -0.183967      4  \n",
       "2 -0.662584  0.691889  0.356801 -0.175259 -0.236449      6  \n",
       "3  0.282612  1.438068  1.058033  0.842981  0.130923      2  \n",
       "4 -0.347519 -0.875088 -1.220973 -0.774223 -0.551339      4  \n",
       "\n",
       "[5 rows x 37 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "eaa224f9-2ff0-441e-bcf4-4bdbe4589584",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    1531\n",
       "6    1508\n",
       "2    1356\n",
       "4     707\n",
       "1     703\n",
       "3     625\n",
       "Name: class, dtype: int64"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df['class'].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "cf8a8403-6b93-47a4-b66e-e1d3046da652",
   "metadata": {},
   "source": [
    "There are five classed, with labels 0-6 (class label 5 is missing)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "bf9242b2-7a0f-4578-baf4-635cae8a731f",
   "metadata": {},
   "source": [
    "We split the data into a proper training set, calibration set and test set"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c1f6e283-c945-43d6-b94d-4f55e52c0579",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_train_cal, df_test = train_test_split(df, test_size = 2000, random_state = 42, shuffle = False)\n",
    "df_proper_train, df_cal = train_test_split(df_train_cal, test_size = 2000, random_state = 42, shuffle = False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "e3d921f2-e4cb-40d7-8b81-d93cbca874fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train = df_train_cal.drop('class', axis=1)\n",
    "y_train = df_train_cal['class']\n",
    "\n",
    "X_proper_train = df_proper_train.drop('class', axis=1)\n",
    "y_proper_train = df_proper_train['class']\n",
    "\n",
    "X_cal = df_cal.drop('class', axis=1)\n",
    "y_cal = df_cal['class']\n",
    "\n",
    "X_test = df_test.drop('class', axis=1)\n",
    "y_test = df_test['class']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "6e1450f2-373f-4464-8143-0da7642e8ccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "lb = LabelBinarizer()\n",
    "y_test_binary = lb.fit_transform(y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "75502f56-6925-491d-9f34-fb5ac650837c",
   "metadata": {},
   "outputs": [],
   "source": [
    "def brier_loss_calc(y_true, prob):\n",
    "    return ((y_true - prob)**2).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "26cec1ce-a36c-40b5-bfc2-0264ad6892ac",
   "metadata": {},
   "source": [
    "## Classifier comparison"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "622e67f7-84c3-4ebb-a765-2ce5e20a56f7",
   "metadata": {},
   "source": [
    "We compare 5 underlying classifiers and two calibration methods (sigmoid and isotonic), both in their inductive and cross-validated format"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2eda5de9-9f96-4734-bf5a-e8e9e5cacb5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "clfs = {}\n",
    "clfs['Naive Bayes'] = GaussianNB()\n",
    "clfs['SVM'] = SVC(probability=True)\n",
    "clfs['RF'] = RandomForestClassifier()\n",
    "clfs['XGB'] = AdaBoostClassifier()\n",
    "clfs['Logistic'] = LogisticRegression(max_iter=10000)\n",
    "clfs['Neural Network'] =  MLPClassifier(max_iter=10000)\n",
    "\n",
    "\n",
    "def run_multiclass_comparison(clf_name, clf):\n",
    "\n",
    "    print(clf_name + ':')\n",
    "    log_loss_list = []\n",
    "    brier_loss_list = []\n",
    "    acc_list = []\n",
    "    ece_list = []\n",
    "    \n",
    "    print('base')\n",
    "    clf.fit(X_train, y_train)\n",
    "    p_pred = clf.predict_proba(X_test)\n",
    "    y_pred = clf.predict(X_test)\n",
    "    acc_list.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_list.append(log_loss(y_test, p_pred))\n",
    "    brier_loss_list.append(brier_loss_calc(y_test_binary, p_pred))\n",
    "    \n",
    "    print('sigmoid')\n",
    "    clf.fit(X_proper_train, y_proper_train)\n",
    "    cal_sigm = CalibratedClassifierCV(clf, method='sigmoid', cv='prefit')\n",
    "    cal_sigm.fit(X_cal, y_cal)\n",
    "    p_pred = cal_sigm.predict_proba(X_test)\n",
    "    y_pred = cal_sigm.predict(X_test)\n",
    "    acc_list.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_list.append(log_loss(y_test, p_pred))\n",
    "    brier_loss_list.append(brier_loss_calc(y_test_binary, p_pred))\n",
    "    \n",
    "    print('isotonic')\n",
    "    cal_iso = CalibratedClassifierCV(clf, method='isotonic', cv='prefit')\n",
    "    cal_iso.fit(X_cal, y_cal)\n",
    "    p_pred = cal_iso.predict_proba(X_test)\n",
    "    y_pred = cal_iso.predict(X_test)\n",
    "    acc_list.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_list.append(log_loss(y_test, p_pred))\n",
    "    brier_loss_list.append(brier_loss_calc(y_test_binary, p_pred))\n",
    "    \n",
    "    print('sigmoid_cv')\n",
    "    cal_sigm_cv = CalibratedClassifierCV(clf, method='sigmoid', cv=5)\n",
    "    cal_sigm_cv.fit(X_train, y_train)\n",
    "    p_pred = cal_sigm_cv.predict_proba(X_test)\n",
    "    y_pred = cal_sigm_cv.predict(X_test)\n",
    "    acc_list.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_list.append(log_loss(y_test, p_pred))\n",
    "    brier_loss_list.append(brier_loss_calc(y_test_binary, p_pred))\n",
    "    \n",
    "    print('isotonic_cv')\n",
    "    cal_iso_cv = CalibratedClassifierCV(clf, method='isotonic', cv=5)\n",
    "    cal_iso_cv.fit(X_train, y_train)\n",
    "    p_pred = cal_iso_cv.predict_proba(X_test)\n",
    "    y_pred = cal_iso_cv.predict(X_test)\n",
    "    acc_list.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_list.append(log_loss(y_test, p_pred))\n",
    "    brier_loss_list.append(brier_loss_calc(y_test_binary, p_pred))\n",
    "    \n",
    "    print('ivap')\n",
    "    va = VennAbersCalibrator(clf, inductive=True, cal_size=0.2, random_state=42)\n",
    "    va.fit(np.asarray(X_train), np.asarray(y_train))\n",
    "    p_pred_va = va.predict_proba(np.array(X_test))\n",
    "    y_pred = va.predict(np.array(X_test), one_hot=False)\n",
    "    acc_list.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_list.append(log_loss(y_test, p_pred_va))\n",
    "    brier_loss_list.append(brier_loss_calc(y_test_binary, p_pred_va))\n",
    "    \n",
    "    print('cvap')\n",
    "    va_cv = VennAbersCalibrator(clf, inductive=False, n_splits=5)\n",
    "    va_cv.fit(np.asarray(X_train), np.asarray(y_train))\n",
    "    p_pred_cv = va_cv.predict_proba(np.asarray(X_test))\n",
    "    y_pred = va_cv.predict(np.array(X_test), one_hot=False)\n",
    "    acc_list.append(accuracy_score(y_test, y_pred))\n",
    "    log_loss_list.append(log_loss(y_test, p_pred_cv))\n",
    "    brier_loss_list.append(brier_loss_calc(y_test_binary, p_pred_cv))\n",
    "\n",
    "    print('')\n",
    "    \n",
    "    df_ll = pd.DataFrame(columns=['Classifier', 'Uncalibrated', 'Platt', 'Isotonic', 'Platt-CV', 'Isotonic-CV', 'IVAP', 'CVAP'])\n",
    "    df_ll.loc[0] =  [clf_name] + log_loss_list\n",
    "    df_bl = pd.DataFrame(columns=['Classifier', 'Uncalibrated', 'Platt', 'Isotonic', 'Platt-CV', 'Isotonic-CV', 'IVAP', 'CVAP'])\n",
    "    df_bl.loc[0] =  [clf_name] + brier_loss_list\n",
    "    df_acc = pd.DataFrame(columns=['Classifier', 'Uncalibrated', 'Platt', 'Isotonic', 'Platt-CV', 'Isotonic-CV', 'IVAP', 'CVAP'])\n",
    "    df_acc.loc[0] =  [clf_name] + acc_list\n",
    "\n",
    "    return df_bl, df_ll, df_acc"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "af282e88-bbb3-4030-baa4-4948a3d829db",
   "metadata": {},
   "source": [
    "We compare accuracy, Brier and log loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "99cb6f80-e9cc-46ae-ba09-44250c6eef6b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Naive Bayes:\n",
      "base\n",
      "sigmoid\n",
      "isotonic\n",
      "sigmoid_cv\n",
      "isotonic_cv\n",
      "ivap\n",
      "cvap\n",
      "\n",
      "SVM:\n",
      "base\n",
      "sigmoid\n",
      "isotonic\n",
      "sigmoid_cv\n",
      "isotonic_cv\n",
      "ivap\n",
      "cvap\n",
      "\n",
      "RF:\n",
      "base\n",
      "sigmoid\n",
      "isotonic\n",
      "sigmoid_cv\n",
      "isotonic_cv\n",
      "ivap\n",
      "cvap\n",
      "\n",
      "XGB:\n",
      "base\n",
      "sigmoid\n",
      "isotonic\n",
      "sigmoid_cv\n",
      "isotonic_cv\n",
      "ivap\n",
      "cvap\n",
      "\n",
      "Logistic:\n",
      "base\n",
      "sigmoid\n",
      "isotonic\n",
      "sigmoid_cv\n",
      "isotonic_cv\n",
      "ivap\n",
      "cvap\n",
      "\n",
      "Neural Network:\n",
      "base\n",
      "sigmoid\n",
      "isotonic\n",
      "sigmoid_cv\n",
      "isotonic_cv\n",
      "ivap\n",
      "cvap\n",
      "\n"
     ]
    }
   ],
   "source": [
    "results_brier = pd.DataFrame()\n",
    "results_log = pd.DataFrame()\n",
    "results_acc = pd.DataFrame()\n",
    "\n",
    "for clf_name in clfs:\n",
    "    scratch_b, scratch_l, scratch_acc = run_multiclass_comparison(clf_name, clfs[clf_name])\n",
    "    results_brier = pd.concat((results_brier, scratch_b), ignore_index=True)\n",
    "    results_log = pd.concat((results_log, scratch_l), ignore_index=True)\n",
    "    results_acc = pd.concat((results_acc, scratch_acc), ignore_index=True)\n",
    "    "
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74c5e18d-6ed4-4eb7-baa6-885b65feea8b",
   "metadata": {},
   "source": [
    "Calibrating can result in a slight improvement in accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "dac8e836-ca42-4ed4-a336-acf35bfba294",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Uncalibrated</th>\n",
       "      <th>Platt</th>\n",
       "      <th>Isotonic</th>\n",
       "      <th>Platt-CV</th>\n",
       "      <th>Isotonic-CV</th>\n",
       "      <th>IVAP</th>\n",
       "      <th>CVAP</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Classifier</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Naive Bayes</th>\n",
       "      <td>0.796</td>\n",
       "      <td>0.792</td>\n",
       "      <td>0.804</td>\n",
       "      <td>0.795</td>\n",
       "      <td>0.806</td>\n",
       "      <td>0.810</td>\n",
       "      <td>0.815</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVM</th>\n",
       "      <td>0.894</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.880</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.893</td>\n",
       "      <td>0.890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RF</th>\n",
       "      <td>0.901</td>\n",
       "      <td>0.892</td>\n",
       "      <td>0.890</td>\n",
       "      <td>0.902</td>\n",
       "      <td>0.902</td>\n",
       "      <td>0.902</td>\n",
       "      <td>0.904</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB</th>\n",
       "      <td>0.750</td>\n",
       "      <td>0.518</td>\n",
       "      <td>0.576</td>\n",
       "      <td>0.740</td>\n",
       "      <td>0.770</td>\n",
       "      <td>0.883</td>\n",
       "      <td>0.887</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic</th>\n",
       "      <td>0.862</td>\n",
       "      <td>0.802</td>\n",
       "      <td>0.822</td>\n",
       "      <td>0.790</td>\n",
       "      <td>0.814</td>\n",
       "      <td>0.864</td>\n",
       "      <td>0.866</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Neural Network</th>\n",
       "      <td>0.894</td>\n",
       "      <td>0.885</td>\n",
       "      <td>0.884</td>\n",
       "      <td>0.900</td>\n",
       "      <td>0.906</td>\n",
       "      <td>0.899</td>\n",
       "      <td>0.898</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Uncalibrated  Platt  Isotonic  Platt-CV  Isotonic-CV   IVAP  \\\n",
       "Classifier                                                                    \n",
       "Naive Bayes            0.796  0.792     0.804     0.795        0.806  0.810   \n",
       "SVM                    0.894  0.880     0.880     0.892        0.890  0.893   \n",
       "RF                     0.901  0.892     0.890     0.902        0.902  0.902   \n",
       "XGB                    0.750  0.518     0.576     0.740        0.770  0.883   \n",
       "Logistic               0.862  0.802     0.822     0.790        0.814  0.864   \n",
       "Neural Network         0.894  0.885     0.884     0.900        0.906  0.899   \n",
       "\n",
       "                 CVAP  \n",
       "Classifier             \n",
       "Naive Bayes     0.815  \n",
       "SVM             0.890  \n",
       "RF              0.904  \n",
       "XGB             0.887  \n",
       "Logistic        0.866  \n",
       "Neural Network  0.898  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_acc.set_index('Classifier', inplace=True)\n",
    "results_acc.round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a2e9793a-c5a3-4b70-93d1-f221906fa649",
   "metadata": {},
   "source": [
    "It generally improves Brier and log loss metrics"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0f905f30-8423-4f42-bc7f-20c998a7794e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Uncalibrated</th>\n",
       "      <th>Platt</th>\n",
       "      <th>Isotonic</th>\n",
       "      <th>Platt-CV</th>\n",
       "      <th>Isotonic-CV</th>\n",
       "      <th>IVAP</th>\n",
       "      <th>CVAP</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Classifier</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Naive Bayes</th>\n",
       "      <td>0.0657</td>\n",
       "      <td>0.0571</td>\n",
       "      <td>0.0480</td>\n",
       "      <td>0.0556</td>\n",
       "      <td>0.0469</td>\n",
       "      <td>0.0449</td>\n",
       "      <td>0.0444</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVM</th>\n",
       "      <td>0.0257</td>\n",
       "      <td>0.0350</td>\n",
       "      <td>0.0309</td>\n",
       "      <td>0.0304</td>\n",
       "      <td>0.0274</td>\n",
       "      <td>0.0273</td>\n",
       "      <td>0.0272</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RF</th>\n",
       "      <td>0.0242</td>\n",
       "      <td>0.0253</td>\n",
       "      <td>0.0251</td>\n",
       "      <td>0.0233</td>\n",
       "      <td>0.0230</td>\n",
       "      <td>0.0253</td>\n",
       "      <td>0.0248</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB</th>\n",
       "      <td>0.0994</td>\n",
       "      <td>0.1025</td>\n",
       "      <td>0.0958</td>\n",
       "      <td>0.0744</td>\n",
       "      <td>0.0613</td>\n",
       "      <td>0.0291</td>\n",
       "      <td>0.0280</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic</th>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0529</td>\n",
       "      <td>0.0481</td>\n",
       "      <td>0.0515</td>\n",
       "      <td>0.0455</td>\n",
       "      <td>0.0317</td>\n",
       "      <td>0.0314</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Neural Network</th>\n",
       "      <td>0.0287</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0278</td>\n",
       "      <td>0.0242</td>\n",
       "      <td>0.0223</td>\n",
       "      <td>0.0251</td>\n",
       "      <td>0.0253</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Uncalibrated   Platt  Isotonic  Platt-CV  Isotonic-CV    IVAP  \\\n",
       "Classifier                                                                      \n",
       "Naive Bayes           0.0657  0.0571    0.0480    0.0556       0.0469  0.0449   \n",
       "SVM                   0.0257  0.0350    0.0309    0.0304       0.0274  0.0273   \n",
       "RF                    0.0242  0.0253    0.0251    0.0233       0.0230  0.0253   \n",
       "XGB                   0.0994  0.1025    0.0958    0.0744       0.0613  0.0291   \n",
       "Logistic              0.0323  0.0529    0.0481    0.0515       0.0455  0.0317   \n",
       "Neural Network        0.0287  0.0323    0.0278    0.0242       0.0223  0.0251   \n",
       "\n",
       "                  CVAP  \n",
       "Classifier              \n",
       "Naive Bayes     0.0444  \n",
       "SVM             0.0272  \n",
       "RF              0.0248  \n",
       "XGB             0.0280  \n",
       "Logistic        0.0314  \n",
       "Neural Network  0.0253  "
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_brier.set_index('Classifier', inplace=True)\n",
    "results_brier.round(4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "8b4b134a-841d-43b6-b399-51cb0f2c8d45",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Uncalibrated</th>\n",
       "      <th>Platt</th>\n",
       "      <th>Isotonic</th>\n",
       "      <th>Platt-CV</th>\n",
       "      <th>Isotonic-CV</th>\n",
       "      <th>IVAP</th>\n",
       "      <th>CVAP</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Classifier</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Naive Bayes</th>\n",
       "      <td>4.0378</td>\n",
       "      <td>0.7704</td>\n",
       "      <td>0.6101</td>\n",
       "      <td>0.7532</td>\n",
       "      <td>0.5960</td>\n",
       "      <td>0.5150</td>\n",
       "      <td>0.5128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>SVM</th>\n",
       "      <td>0.2805</td>\n",
       "      <td>0.4341</td>\n",
       "      <td>0.4587</td>\n",
       "      <td>0.3678</td>\n",
       "      <td>0.3335</td>\n",
       "      <td>0.3383</td>\n",
       "      <td>0.3356</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>RF</th>\n",
       "      <td>0.2706</td>\n",
       "      <td>0.2967</td>\n",
       "      <td>0.3613</td>\n",
       "      <td>0.2730</td>\n",
       "      <td>0.2755</td>\n",
       "      <td>0.3031</td>\n",
       "      <td>0.2973</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>XGB</th>\n",
       "      <td>1.1760</td>\n",
       "      <td>1.2685</td>\n",
       "      <td>1.2437</td>\n",
       "      <td>0.9034</td>\n",
       "      <td>0.7280</td>\n",
       "      <td>0.3522</td>\n",
       "      <td>0.3439</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Logistic</th>\n",
       "      <td>0.3564</td>\n",
       "      <td>0.6104</td>\n",
       "      <td>0.6166</td>\n",
       "      <td>0.5951</td>\n",
       "      <td>0.5857</td>\n",
       "      <td>0.3739</td>\n",
       "      <td>0.3701</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Neural Network</th>\n",
       "      <td>0.4091</td>\n",
       "      <td>0.4403</td>\n",
       "      <td>0.4782</td>\n",
       "      <td>0.3199</td>\n",
       "      <td>0.2713</td>\n",
       "      <td>0.3072</td>\n",
       "      <td>0.3112</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                Uncalibrated   Platt  Isotonic  Platt-CV  Isotonic-CV    IVAP  \\\n",
       "Classifier                                                                      \n",
       "Naive Bayes           4.0378  0.7704    0.6101    0.7532       0.5960  0.5150   \n",
       "SVM                   0.2805  0.4341    0.4587    0.3678       0.3335  0.3383   \n",
       "RF                    0.2706  0.2967    0.3613    0.2730       0.2755  0.3031   \n",
       "XGB                   1.1760  1.2685    1.2437    0.9034       0.7280  0.3522   \n",
       "Logistic              0.3564  0.6104    0.6166    0.5951       0.5857  0.3739   \n",
       "Neural Network        0.4091  0.4403    0.4782    0.3199       0.2713  0.3072   \n",
       "\n",
       "                  CVAP  \n",
       "Classifier              \n",
       "Naive Bayes     0.5128  \n",
       "SVM             0.3356  \n",
       "RF              0.2973  \n",
       "XGB             0.3439  \n",
       "Logistic        0.3701  \n",
       "Neural Network  0.3112  "
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_log.set_index('Classifier', inplace=True)\n",
    "results_log.round(4)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4d773c6e-a371-49cb-8c5f-6bf250217ffc",
   "metadata": {},
   "source": [
    "**Mean accuracy**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5f906a20-3097-4ba2-9b4a-e281110b38fe",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Uncalibrated    0.849417\n",
       "Platt           0.794583\n",
       "Isotonic        0.809167\n",
       "Platt-CV        0.836250\n",
       "Isotonic-CV     0.847917\n",
       "IVAP            0.875167\n",
       "CVAP            0.876583\n",
       "dtype: float64"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_acc.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "021427e2-ca07-4f52-9fec-40ac1bd5b84d",
   "metadata": {},
   "source": [
    "**Average Brier and log loss ranking**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "3e4af628-c204-493f-90a0-5e0534577db0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Uncalibrated    4.333333\n",
       "Platt           6.833333\n",
       "Isotonic        5.000000\n",
       "Platt-CV        4.000000\n",
       "Isotonic-CV     2.666667\n",
       "IVAP            3.000000\n",
       "CVAP            2.166667\n",
       "dtype: float64"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_brier.rank(axis=1).mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "808a22db-b7b4-454a-87c4-96e38886ce5d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Uncalibrated    3.333333\n",
       "Platt           5.833333\n",
       "Isotonic        6.333333\n",
       "Platt-CV        4.166667\n",
       "Isotonic-CV     2.666667\n",
       "IVAP            3.166667\n",
       "CVAP            2.500000\n",
       "dtype: float64"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_log.rank(axis=1).mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4c684e55-19e2-4153-a9b4-994c3b0fed6d",
   "metadata": {},
   "source": [
    "**Mean Brier and log loss scores**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "f306814c-a6d8-4b60-beca-ad995df4f8b0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Uncalibrated    0.045989\n",
       "Platt           0.050838\n",
       "Isotonic        0.045943\n",
       "Platt-CV        0.043247\n",
       "Isotonic-CV     0.037716\n",
       "IVAP            0.030563\n",
       "CVAP            0.030184\n",
       "dtype: float64"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_brier.mean()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7db3752b-7352-4b94-8510-d1be7dfe1cf8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Uncalibrated    1.088424\n",
       "Platt           0.636729\n",
       "Isotonic        0.628093\n",
       "Platt-CV        0.535397\n",
       "Isotonic-CV     0.464994\n",
       "IVAP            0.364935\n",
       "CVAP            0.361827\n",
       "dtype: float64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "results_log.mean()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f7828698-70f9-4b11-bc41-14d72bf1d9ca",
   "metadata": {},
   "source": [
    "Overall, in this example, IVAP and CVAP calibrators achieve highest overall accuracy and lowest Brier and log loss errors"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "4b1e8d31-3298-4cf9-8313-f8853134ca2c",
   "metadata": {},
   "source": [
    "## Multiclass implemetation of pre-fitted classifiers"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f5cce6a3-915f-44d8-a3b7-44089c413a99",
   "metadata": {},
   "source": [
    "In standard IVAP and CVAP multi-class calibration, we consider a collection of binary\n",
    "classification problems and then combine their solutions (when solutions include pairwise\n",
    "class probabilities) to obtain multi-class probabilities (see e.g. *Manokhin In Conformal and Probabilistic Prediction and Applications, pp. 228-240. PMLR, 2017.*). For some classifiers (e.g. Deep Neural Networks) it may not be practical to modify the problem to generate a set of one-vs-one probability outputs. In this case, we can use the calibration set multi-class probabilities to calibrate the outputs in a similar way, by converting them into equivalent one-vs-one or one-vs-all outputs first for binary class pair combinations in the calibrations set and applying the VennABERS procedure directly to them. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0c6df54b-cf20-42dd-a44b-80b34be78dc5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>RF</th>\n",
       "      <th>VA_one_vs_one</th>\n",
       "      <th>VA_one_vs_all</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Brier loss</th>\n",
       "      <td>0.064</td>\n",
       "      <td>0.062</td>\n",
       "      <td>0.061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>log loss</th>\n",
       "      <td>0.730</td>\n",
       "      <td>0.618</td>\n",
       "      <td>0.599</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>accuracy</th>\n",
       "      <td>0.790</td>\n",
       "      <td>0.789</td>\n",
       "      <td>0.789</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               RF  VA_one_vs_one  VA_one_vs_all\n",
       "Brier loss  0.064          0.062          0.061\n",
       "log loss    0.730          0.618          0.599\n",
       "accuracy    0.790          0.789          0.789"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# create a synthetic dataset\n",
    "\n",
    "n_features = 10\n",
    "rand_seed = 0\n",
    "n_samples = 10000\n",
    "\n",
    "X, y = make_classification(\n",
    "    n_classes=5,\n",
    "    n_samples=n_samples,\n",
    "    n_clusters_per_class=2,\n",
    "    n_features=n_features,\n",
    "    n_informative=int(n_features/2),\n",
    "    n_redundant=int(n_features/4),\n",
    "    random_state=rand_seed)\n",
    "\n",
    "#  split into a proper training, calibration and test set\n",
    "\n",
    "X_train, X_test, y_train, y_test = train_test_split(\n",
    "    X, y, test_size=0.5, random_state=rand_seed)\n",
    "\n",
    "X_train_proper, X_cal, y_train_proper, y_cal = train_test_split(\n",
    "    X_train, y_train, test_size=0.2, shuffle=False)\n",
    "\n",
    "lb = LabelBinarizer()\n",
    "y_test_one_hot = lb.fit_transform(y_test)\n",
    "\n",
    "# fit a Random Forest classifier to the proper training set \n",
    "# and generate calibration and test set probabilities\n",
    "\n",
    "clf = RandomForestClassifier(random_state=rand_seed)\n",
    "\n",
    "clf.fit(X_train_proper, y_train_proper)\n",
    "p_cal = clf.predict_proba(X_cal)\n",
    "p_test = clf.predict_proba(X_test)\n",
    "y_pred = clf.predict(X_test)\n",
    "\n",
    "# fit a VennAbers calibrator using the calibration set and generate calibrated test set \n",
    "# probabilities (note this is done using a single command) and corresponding predictions\n",
    "# please also note you can specify one_vs_one or one_vs_all settings for VennABERS calibration\n",
    "\n",
    "va = VennAbersCalibrator()\n",
    "p_prime_one_vs_one, p0p1_one_vs_one = va.predict_proba(\n",
    "    p_cal=p_cal, y_cal=y_cal, p_test=p_test, p0_p1_output=True, va_type = 'one_vs_one')\n",
    "y_prime_one_vs_one = va.predict(\n",
    "    p_cal=p_cal, y_cal=y_cal, p_test=p_test, one_hot=False)\n",
    "\n",
    "\n",
    "va = VennAbersCalibrator()\n",
    "p_prime_one_vs_all, p0p1_one_vs_all = va.predict_proba(\n",
    "    p_cal=p_cal, y_cal=y_cal, p_test=p_test, p0_p1_output=True, va_type = 'one_vs_all')\n",
    "y_prime_one_vs_all = va.predict(\n",
    "    p_cal=p_cal, y_cal=y_cal, p_test=p_test, one_hot=False)\n",
    "\n",
    "# compare log loss, Brier loss and accuracy \n",
    "\n",
    "stat_list = {}\n",
    "stat_list['RF'] = [\n",
    "    brier_loss_calc(y_test_one_hot, p_test),\n",
    "    log_loss(y_test, p_test),\n",
    "    accuracy_score(y_test, y_pred)]\n",
    "\n",
    "stat_list['VA_one_vs_one'] = [\n",
    "    brier_loss_calc(y_test_one_hot, p_prime_one_vs_one),\n",
    "    log_loss(y_test, p_prime_one_vs_one),\n",
    "    accuracy_score(y_test, y_prime_one_vs_one)]\n",
    "\n",
    "stat_list['VA_one_vs_all'] = [\n",
    "    brier_loss_calc(y_test_one_hot, p_prime_one_vs_all),\n",
    "    log_loss(y_test, p_prime_one_vs_all),\n",
    "    accuracy_score(y_test, y_prime_one_vs_all)]\n",
    "    \n",
    "stat_df = pd.DataFrame(stat_list, index = ['Brier loss', 'log loss', 'accuracy'])\n",
    "stat_df.round(3)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "88999763-a342-4ce1-bf52-9fafc10fbf17",
   "metadata": {},
   "source": [
    "In this example the VennABERS calibrated RF multiclass probabilities yield a lower Brier and log loss metrics with similar accuracy"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}