tbl1_SDT-AT_Original.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#AA+OF\" data-toc-modified-id=\"AA+OF-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>AA+OF</a></span></li><li><span><a href=\"#AA\" data-toc-modified-id=\"AA-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>AA</a></span></li><li><span><a href=\"#PA+OF\" data-toc-modified-id=\"PA+OF-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>PA+OF</a></span></li><li><span><a href=\"#PA\" data-toc-modified-id=\"PA-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>PA</a></span></li><li><span><a href=\"#NA+OF\" data-toc-modified-id=\"NA+OF-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>NA+OF</a></span></li><li><span><a href=\"#NA\" data-toc-modified-id=\"NA-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>NA</a></span></li><li><span><a href=\"#OF\" data-toc-modified-id=\"OF-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>OF</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:18:33.329825Z",
     "start_time": "2023-06-13T08:18:31.936605Z"
    }
   },
   "outputs": [],
   "source": [
    "from sdt import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:18:34.226073Z",
     "start_time": "2023-06-13T08:18:34.218556Z"
    }
   },
   "outputs": [],
   "source": [
    "av_mode = [\n",
    "    'abs', 'mean', 'min', 'abs*mean', 'max', 'euclidean',\n",
    "    'mahalanobis', 'cos', 'adj_cos', 'manhattan', 'wmd', 'wrd'\n",
    "]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:18:45.452799Z",
     "start_time": "2023-06-13T08:18:45.444962Z"
    }
   },
   "outputs": [],
   "source": [
    "allScore, firstScore, secondScore, thirdScore = pickle.load(\n",
    "    open('data/av_data/original/aa+of.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:18:57.500815Z",
     "start_time": "2023-06-13T08:18:54.859496Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 79.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3881\n",
      "    precision-macro:       0.3514\n",
      "    recall-macro:          0.3583\n",
      "    f1-marcro:             0.3536\n",
      "    spearman-label:        0.1768 0.0060\n",
      "    av magnitude\n",
      "        spearman_all:      0.1768 0.0121\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='1',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:19:26.495904Z",
     "start_time": "2023-06-13T08:19:23.640426Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 72.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3980\n",
      "    precision-macro:       0.3523\n",
      "    recall-macro:          0.3568\n",
      "    f1-marcro:             0.3541\n",
      "    spearman-label:        0.1850 0.0043\n",
      "    av magnitude\n",
      "        spearman_all:      -0.1850 0.0086\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='2',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:20:36.845097Z",
     "start_time": "2023-06-13T08:20:34.416181Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 28.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3529\n",
      "    precision-macro:       0.1176\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1739\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:07.793455Z",
     "start_time": "2023-06-13T08:21:05.563454Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2794\n",
      "    precision-macro:       0.1528\n",
      "    recall-macro:          0.2738\n",
      "    f1-marcro:             0.1923\n",
      "    spearman-label:        -0.3985 0.9996\n",
      "    av magnitude\n",
      "        spearman_all:      0.3985 0.0008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:14.202032Z",
     "start_time": "2023-06-13T08:21:11.999029Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.43it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3971\n",
      "    precision-macro:       0.3342\n",
      "    recall-macro:          0.3350\n",
      "    f1-marcro:             0.3327\n",
      "    spearman-label:        0.1357 0.1350\n",
      "    av magnitude\n",
      "        spearman_all:      0.1357 0.2699\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:22.647983Z",
     "start_time": "2023-06-13T08:21:20.427630Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3971\n",
      "    precision-macro:       0.3671\n",
      "    recall-macro:          0.3657\n",
      "    f1-marcro:             0.3657\n",
      "    spearman-label:        0.1750 0.0767\n",
      "    av magnitude\n",
      "        spearman_all:      -0.1750 0.1534\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:28.825249Z",
     "start_time": "2023-06-13T08:21:26.595627Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3692\n",
      "    precision-macro:       0.3623\n",
      "    recall-macro:          0.3507\n",
      "    f1-marcro:             0.3508\n",
      "    spearman-label:        0.1490 0.1181\n",
      "    av magnitude\n",
      "        spearman_all:      0.1490 0.2363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:35.282127Z",
     "start_time": "2023-06-13T08:21:32.988831Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 29.79it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2923\n",
      "    precision-macro:       0.0974\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1508\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:38.945714Z",
     "start_time": "2023-06-13T08:21:38.938754Z"
    }
   },
   "outputs": [],
   "source": [
    "allScore, firstScore, secondScore, thirdScore = pickle.load(\n",
    "    open('data/av_data/original/aa.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:47.723408Z",
     "start_time": "2023-06-13T08:21:44.923984Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 75.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4129\n",
      "    precision-macro:       0.3745\n",
      "    recall-macro:          0.3799\n",
      "    f1-marcro:             0.3754\n",
      "    spearman-label:        0.1816 0.0049\n",
      "    av magnitude\n",
      "        spearman_all:      0.1816 0.0099\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='1',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:54.215477Z",
     "start_time": "2023-06-13T08:21:51.575954Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 79.85it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3035\n",
      "    precision-macro:       0.1048\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1595\n",
      "    spearman-label:        -0.2718 1.0000\n",
      "    av magnitude\n",
      "        spearman_all:      0.2718 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='2',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:21:59.738995Z",
     "start_time": "2023-06-13T08:21:57.529893Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3529\n",
      "    precision-macro:       0.1270\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1839\n",
      "    spearman-label:        -0.3552 0.9985\n",
      "    av magnitude\n",
      "        spearman_all:      -0.3552 0.0030\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:22:06.005903Z",
     "start_time": "2023-06-13T08:22:03.764192Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3529\n",
      "    precision-macro:       0.1176\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1739\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:22:16.217124Z",
     "start_time": "2023-06-13T08:22:14.024273Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4706\n",
      "    precision-macro:       0.4290\n",
      "    recall-macro:          0.4345\n",
      "    f1-marcro:             0.4304\n",
      "    spearman-label:        0.2409 0.0239\n",
      "    av magnitude\n",
      "        spearman_all:      0.2409 0.0478\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:22:24.289304Z",
     "start_time": "2023-06-13T08:22:22.059822Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2647\n",
      "    precision-macro:       0.0882\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1395\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:22:31.780442Z",
     "start_time": "2023-06-13T08:22:29.567087Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3692\n",
      "    precision-macro:       0.3356\n",
      "    recall-macro:          0.3325\n",
      "    f1-marcro:             0.3339\n",
      "    spearman-label:        0.2019 0.0534\n",
      "    av magnitude\n",
      "        spearman_all:      0.2019 0.1068\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:22:49.857995Z",
     "start_time": "2023-06-13T08:22:47.410131Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 27.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2923\n",
      "    precision-macro:       0.0974\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1508\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:22:58.504416Z",
     "start_time": "2023-06-13T08:22:58.494770Z"
    }
   },
   "outputs": [],
   "source": [
    "allScore, firstScore, secondScore, thirdScore = pickle.load(\n",
    "    open('data/av_data/original/pa+of.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:23:06.830149Z",
     "start_time": "2023-06-13T08:23:04.291628Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 83.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3333\n",
      "    precision-macro:       0.3083\n",
      "    recall-macro:          0.3069\n",
      "    f1-marcro:             0.3074\n",
      "    spearman-label:        0.0326 0.3228\n",
      "    av magnitude\n",
      "        spearman_all:      0.0326 0.6456\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='1',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:23:19.453719Z",
     "start_time": "2023-06-13T08:23:16.817556Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 80.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3035\n",
      "    precision-macro:       0.1012\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1552\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='2',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:23:28.318744Z",
     "start_time": "2023-06-13T08:23:26.100401Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3088\n",
      "    precision-macro:       0.1207\n",
      "    recall-macro:          0.2917\n",
      "    f1-marcro:             0.1707\n",
      "    spearman-label:        -0.3641 0.9989\n",
      "    av magnitude\n",
      "        spearman_all:      -0.3641 0.0023\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:23:42.026502Z",
     "start_time": "2023-06-13T08:23:39.798301Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3529\n",
      "    precision-macro:       0.1212\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1778\n",
      "    spearman-label:        -0.2580 0.9832\n",
      "    av magnitude\n",
      "        spearman_all:      0.2580 0.0337\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:23:48.517045Z",
     "start_time": "2023-06-13T08:23:46.385411Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 33.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3971\n",
      "    precision-macro:       0.3596\n",
      "    recall-macro:          0.3547\n",
      "    f1-marcro:             0.3568\n",
      "    spearman-label:        0.1539 0.1052\n",
      "    av magnitude\n",
      "        spearman_all:      0.1539 0.2103\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:24:47.619024Z",
     "start_time": "2023-06-13T08:24:45.380297Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2647\n",
      "    precision-macro:       0.0896\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1412\n",
      "    spearman-label:        -0.1762 0.9247\n",
      "    av magnitude\n",
      "        spearman_all:      0.1762 0.1506\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:24:53.316942Z",
     "start_time": "2023-06-13T08:24:51.071033Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3692\n",
      "    precision-macro:       0.3498\n",
      "    recall-macro:          0.3386\n",
      "    f1-marcro:             0.3427\n",
      "    spearman-label:        0.1978 0.0572\n",
      "    av magnitude\n",
      "        spearman_all:      0.1978 0.1143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:24:58.627041Z",
     "start_time": "2023-06-13T08:24:56.401649Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3077\n",
      "    precision-macro:       0.2651\n",
      "    recall-macro:          0.2682\n",
      "    f1-marcro:             0.2646\n",
      "    spearman-label:        0.0636 0.3074\n",
      "    av magnitude\n",
      "        spearman_all:      -0.0636 0.6148\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:00.370593Z",
     "start_time": "2023-06-13T08:25:00.360046Z"
    }
   },
   "outputs": [],
   "source": [
    "allScore, firstScore, secondScore, thirdScore = pickle.load(\n",
    "    open('data/av_data/original/pa.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:09.632220Z",
     "start_time": "2023-06-13T08:25:07.029444Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 82.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4129\n",
      "    precision-macro:       0.3842\n",
      "    recall-macro:          0.3856\n",
      "    f1-marcro:             0.3830\n",
      "    spearman-label:        0.1416 0.0224\n",
      "    av magnitude\n",
      "        spearman_all:      0.1416 0.0449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='1',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:14.780501Z",
     "start_time": "2023-06-13T08:25:12.098303Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 78.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3035\n",
      "    precision-macro:       0.1012\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1552\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='2',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:22.304411Z",
     "start_time": "2023-06-13T08:25:20.091270Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4412\n",
      "    precision-macro:       0.3926\n",
      "    recall-macro:          0.3937\n",
      "    f1-marcro:             0.3901\n",
      "    spearman-label:        0.1738 0.0782\n",
      "    av magnitude\n",
      "        spearman_all:      0.1738 0.1565\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:27.944497Z",
     "start_time": "2023-06-13T08:25:25.835128Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 33.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3529\n",
      "    precision-macro:       0.1176\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1739\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:34.581538Z",
     "start_time": "2023-06-13T08:25:32.338052Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2500\n",
      "    precision-macro:       0.0899\n",
      "    recall-macro:          0.3148\n",
      "    f1-marcro:             0.1399\n",
      "    spearman-label:        -0.2864 0.9911\n",
      "    av magnitude\n",
      "        spearman_all:      -0.2864 0.0179\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:40.771321Z",
     "start_time": "2023-06-13T08:25:38.543364Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3971\n",
      "    precision-macro:       0.3161\n",
      "    recall-macro:          0.3396\n",
      "    f1-marcro:             0.3263\n",
      "    spearman-label:        0.1912 0.0591\n",
      "    av magnitude\n",
      "        spearman_all:      -0.1912 0.1183\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:25:45.898817Z",
     "start_time": "2023-06-13T08:25:43.754353Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 32.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3385\n",
      "    precision-macro:       0.2757\n",
      "    recall-macro:          0.2973\n",
      "    f1-marcro:             0.2806\n",
      "    spearman-label:        -0.0258 0.5808\n",
      "    av magnitude\n",
      "        spearman_all:      -0.0258 0.8385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:26:01.530131Z",
     "start_time": "2023-06-13T08:25:59.318793Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2923\n",
      "    precision-macro:       0.0974\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1508\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:26:03.223224Z",
     "start_time": "2023-06-13T08:26:03.212448Z"
    }
   },
   "outputs": [],
   "source": [
    "allScore, firstScore, secondScore, thirdScore = pickle.load(\n",
    "    open('data/av_data/original/na+of.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:26:10.248275Z",
     "start_time": "2023-06-13T08:26:07.613847Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 79.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2239\n",
      "    precision-macro:       0.2024\n",
      "    recall-macro:          0.1895\n",
      "    f1-marcro:             0.1917\n",
      "    spearman-label:        -0.1204 0.9557\n",
      "    av magnitude\n",
      "        spearman_all:      -0.1204 0.0887\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='1',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:26:15.846501Z",
     "start_time": "2023-06-13T08:26:13.175686Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 78.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3035\n",
      "    precision-macro:       0.1012\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1552\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='2',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:26:26.863860Z",
     "start_time": "2023-06-13T08:26:24.547142Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3529\n",
      "    precision-macro:       0.1176\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1739\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:26:34.012466Z",
     "start_time": "2023-06-13T08:26:31.807435Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3235\n",
      "    precision-macro:       0.2179\n",
      "    recall-macro:          0.2889\n",
      "    f1-marcro:             0.2385\n",
      "    spearman-label:        -0.3397 0.9977\n",
      "    av magnitude\n",
      "        spearman_all:      0.3397 0.0046\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:27:04.306282Z",
     "start_time": "2023-06-13T08:27:01.980004Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 30.91it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3676\n",
      "    precision-macro:       0.3072\n",
      "    recall-macro:          0.3671\n",
      "    f1-marcro:             0.2791\n",
      "    spearman-label:        0.1865 0.0639\n",
      "    av magnitude\n",
      "        spearman_all:      0.1865 0.1278\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:27:16.623076Z",
     "start_time": "2023-06-13T08:27:14.367821Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2647\n",
      "    precision-macro:       0.0882\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1395\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:27:31.971072Z",
     "start_time": "2023-06-13T08:27:29.716119Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3538\n",
      "    precision-macro:       0.3767\n",
      "    recall-macro:          0.3637\n",
      "    f1-marcro:             0.3582\n",
      "    spearman-label:        0.4037 0.0004\n",
      "    av magnitude\n",
      "        spearman_all:      0.4037 0.0009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:27:49.126701Z",
     "start_time": "2023-06-13T08:27:46.957681Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 32.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2923\n",
      "    precision-macro:       0.0990\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1526\n",
      "    spearman-label:        -0.1716 0.9141\n",
      "    av magnitude\n",
      "        spearman_all:      0.1716 0.1717\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:27:54.839568Z",
     "start_time": "2023-06-13T08:27:54.832939Z"
    }
   },
   "outputs": [],
   "source": [
    "allScore, firstScore, secondScore, thirdScore = pickle.load(\n",
    "    open('data/av_data/original/na.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:28:09.697806Z",
     "start_time": "2023-06-13T08:28:07.034699Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 80.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4129\n",
      "    precision-macro:       0.4058\n",
      "    recall-macro:          0.4119\n",
      "    f1-marcro:             0.3452\n",
      "    spearman-label:        0.1685 0.0084\n",
      "    av magnitude\n",
      "        spearman_all:      0.1685 0.0168\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='1',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:28:16.741440Z",
     "start_time": "2023-06-13T08:28:14.021609Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 77.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3035\n",
      "    precision-macro:       0.1012\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1552\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='2',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:28:25.205846Z",
     "start_time": "2023-06-13T08:28:22.897306Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 30.84it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4118\n",
      "    precision-macro:       0.3296\n",
      "    recall-macro:          0.3472\n",
      "    f1-marcro:             0.3359\n",
      "    spearman-label:        0.0272 0.4129\n",
      "    av magnitude\n",
      "        spearman_all:      0.0272 0.8258\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:28:31.538279Z",
     "start_time": "2023-06-13T08:28:29.245914Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4559\n",
      "    precision-macro:       0.4000\n",
      "    recall-macro:          0.4060\n",
      "    f1-marcro:             0.4020\n",
      "    spearman-label:        0.0828 0.2509\n",
      "    av magnitude\n",
      "        spearman_all:      -0.0828 0.5019\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:28:38.071837Z",
     "start_time": "2023-06-13T08:28:35.754995Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2941\n",
      "    precision-macro:       0.2240\n",
      "    recall-macro:          0.2832\n",
      "    f1-marcro:             0.2232\n",
      "    spearman-label:        -0.0105 0.5337\n",
      "    av magnitude\n",
      "        spearman_all:      -0.0105 0.9325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:28:44.437992Z",
     "start_time": "2023-06-13T08:28:42.122358Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.16it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2647\n",
      "    precision-macro:       0.0882\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1395\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:28:52.318408Z",
     "start_time": "2023-06-13T08:28:49.994596Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 29.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4462\n",
      "    precision-macro:       0.4489\n",
      "    recall-macro:          0.4366\n",
      "    f1-marcro:             0.4410\n",
      "    spearman-label:        0.4245 0.0002\n",
      "    av magnitude\n",
      "        spearman_all:      0.4245 0.0004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:29:05.356056Z",
     "start_time": "2023-06-13T08:29:02.921826Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 28.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2923\n",
      "    precision-macro:       0.0974\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1508\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:29:54.747306Z",
     "start_time": "2023-06-13T08:29:54.737428Z"
    }
   },
   "outputs": [],
   "source": [
    "allScore, firstScore, secondScore, thirdScore = pickle.load(\n",
    "    open('data/av_data/original/of.pkl', 'rb'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:01.922303Z",
     "start_time": "2023-06-13T08:29:59.119944Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 75.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2338\n",
      "    precision-macro:       0.0779\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1263\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='1',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:10.163781Z",
     "start_time": "2023-06-13T08:30:07.475333Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "201it [00:02, 79.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3930\n",
      "    precision-macro:       0.2459\n",
      "    recall-macro:          0.3533\n",
      "    f1-marcro:             0.2888\n",
      "    spearman-label:        0.0570 0.2106\n",
      "    av magnitude\n",
      "        spearman_all:      -0.0570 0.4212\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log = nested_cv([allScore, allTrueLabel],\n",
    "                allPredLabel, \n",
    "                h='2',\n",
    "                mode=\"original\",\n",
    "                model=sdt_model,\n",
    "                av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:18.428552Z",
     "start_time": "2023-06-13T08:30:16.110327Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.2059\n",
      "    precision-macro:       0.0686\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.1138\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 61,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:24.549212Z",
     "start_time": "2023-06-13T08:30:22.101023Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 29.54it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3824\n",
      "    precision-macro:       0.2367\n",
      "    recall-macro:          0.3651\n",
      "    f1-marcro:             0.2854\n",
      "    spearman-label:        0.0990 0.2110\n",
      "    av magnitude\n",
      "        spearman_all:      -0.0990 0.4219\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log1 = nested_cv([firstScore, firstTrueLabel],\n",
    "                 firstPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 62,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:31.768193Z",
     "start_time": "2023-06-13T08:30:29.426269Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 31.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4559\n",
      "    precision-macro:       0.2930\n",
      "    recall-macro:          0.4032\n",
      "    f1-marcro:             0.3341\n",
      "    spearman-label:        0.1824 0.0683\n",
      "    av magnitude\n",
      "        spearman_all:      0.1824 0.1366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:38.726045Z",
     "start_time": "2023-06-13T08:30:36.497685Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "68it [00:02, 32.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4559\n",
      "    precision-macro:       0.2930\n",
      "    recall-macro:          0.4032\n",
      "    f1-marcro:             0.3341\n",
      "    spearman-label:        0.1824 0.0683\n",
      "    av magnitude\n",
      "        spearman_all:      -0.1824 0.1366\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log2 = nested_cv([secondScore, secondTrueLabel],\n",
    "                 secondPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:48.400848Z",
     "start_time": "2023-06-13T08:30:46.145062Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.3692\n",
      "    precision-macro:       0.2692\n",
      "    recall-macro:          0.3732\n",
      "    f1-marcro:             0.2894\n",
      "    spearman-label:        0.1104 0.1906\n",
      "    av magnitude\n",
      "        spearman_all:      0.1104 0.3813\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='1',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T08:30:54.993267Z",
     "start_time": "2023-06-13T08:30:52.720784Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "65it [00:02, 30.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV\n",
      "    accuracy:              0.4462\n",
      "    precision-macro:       0.1487\n",
      "    recall-macro:          0.3333\n",
      "    f1-marcro:             0.2057\n",
      "    spearman-label:        nan nan\n",
      "    av magnitude\n",
      "        spearman_all:      nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "log3 = nested_cv([thirdScore, thirdTrueLabel],\n",
    "                 thirdPredLabel,\n",
    "                 h='2',\n",
    "                 mode=\"original\",\n",
    "                 model=sdt_model,\n",
    "                 av_mode=av_mode)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}