tbl1_SDT-AT_PLM-tf.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#AA+OF\" data-toc-modified-id=\"AA+OF-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>AA+OF</a></span></li><li><span><a href=\"#AA\" data-toc-modified-id=\"AA-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>AA</a></span></li><li><span><a href=\"#PA+OF\" data-toc-modified-id=\"PA+OF-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>PA+OF</a></span></li><li><span><a href=\"#PA\" data-toc-modified-id=\"PA-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>PA</a></span></li><li><span><a href=\"#NA+OF\" data-toc-modified-id=\"NA+OF-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>NA+OF</a></span></li><li><span><a href=\"#NA\" data-toc-modified-id=\"NA-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>NA</a></span></li><li><span><a href=\"#OF\" data-toc-modified-id=\"OF-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>OF</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T09:55:05.235957Z",
     "start_time": "2023-06-15T09:55:03.575314Z"
    }
   },
   "outputs": [],
   "source": [
    "from sdt import *\n",
    "from evaluate import *\n",
    "\n",
    "model = 'plm'"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T11:09:25.153247Z",
     "start_time": "2023-06-14T11:09:25.144402Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_aa+of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T12:27:26.884628Z",
     "start_time": "2023-06-14T11:09:30.355152Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 162/162 [1:17:56<00:00, 28.87s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         hfl/rbt3\n",
      "    Label:       0.2384 0.0003\n",
      "    AV magnitude\n",
      "        all:     0.2384 0.0007\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T20:11:01.961092Z",
     "start_time": "2023-06-14T18:50:30.411418Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 162/162 [1:20:31<00:00, 29.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         google/BERT-base-multi\n",
      "    Label:       0.2837 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.2837 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T20:28:02.197497Z",
     "start_time": "2023-06-14T20:11:01.972309Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [17:00<00:00,  6.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         uer/roberta-medium-word-chinese-cluecorpussmall\n",
      "    Label:       0.4113 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.4113 0.0005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T20:45:18.151320Z",
     "start_time": "2023-06-14T20:28:02.207701Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [17:15<00:00,  6.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         ZhuiyiTechnology/mT5-base\n",
      "    Label:       0.3377 0.0024\n",
      "    AV magnitude\n",
      "        all:     -0.3377 0.0049\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T21:02:16.598554Z",
     "start_time": "2023-06-14T20:45:18.162687Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [16:58<00:00,  6.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         hfl/chinese-electra-180g-small-ex-discriminator\n",
      "    Label:       0.4292 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.4292 0.0003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T21:19:30.700587Z",
     "start_time": "2023-06-14T21:02:16.607503Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [17:14<00:00,  6.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         uer/roberta-base-finetuned-ifeng-chinese\n",
      "    Label:       0.4375 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4375 0.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T21:35:57.885070Z",
     "start_time": "2023-06-14T21:19:30.711610Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [16:27<00:00,  6.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         brightmart/RoBERTa-L6\n",
      "    Label:       0.4807 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4807 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T21:52:41.499119Z",
     "start_time": "2023-06-14T21:35:57.895755Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [16:43<00:00,  6.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         uer/chinese_roberta_L-6_H-128\n",
      "    Label:       0.4807 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.4807 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd', model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T21:52:41.510992Z",
     "start_time": "2023-06-14T21:52:41.508992Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_aa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T23:10:29.636434Z",
     "start_time": "2023-06-14T21:52:41.522337Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 162/162 [1:17:48<00:00, 28.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         Yaxin/ernie_1.0_skep_large_ch\n",
      "    Label:       0.2578 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.2578 0.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T00:30:51.282113Z",
     "start_time": "2023-06-14T23:10:29.650753Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 162/162 [1:20:21<00:00, 29.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         uer/roberta-base-chinese-extractive-qa\n",
      "    Label:       0.2879 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.2879 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T00:47:50.030889Z",
     "start_time": "2023-06-15T00:30:51.291887Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [16:58<00:00,  6.29s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         uer/roberta-medium-word-chinese-cluecorpussmall\n",
      "    Label:       0.4639 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4639 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T01:05:03.776369Z",
     "start_time": "2023-06-15T00:47:50.040368Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [17:13<00:00,  6.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         google/BERT-base\n",
      "    Label:       0.4300 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4300 0.0003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T01:21:57.898360Z",
     "start_time": "2023-06-15T01:05:03.786500Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [16:54<00:00,  6.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         hfl/chinese-electra-small-ex-discriminator\n",
      "    Label:       0.4173 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.4173 0.0004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T01:39:11.344219Z",
     "start_time": "2023-06-15T01:21:57.908224Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [17:13<00:00,  6.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         clue/roberta_chinese_clue_tiny\n",
      "    Label:       0.3845 0.0006\n",
      "    AV magnitude\n",
      "        all:     -0.3845 0.0012\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T01:55:37.696553Z",
     "start_time": "2023-06-15T01:39:11.353830Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [16:26<00:00,  6.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         hfl/chinese-legal-electra-small-discriminator\n",
      "    Label:       0.4678 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4678 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T02:12:19.605478Z",
     "start_time": "2023-06-15T01:55:37.707131Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [16:41<00:00,  6.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         uer/chinese_roberta_L-6_H-128\n",
      "    Label:       0.4974 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.4974 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd', model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T09:55:16.265533Z",
     "start_time": "2023-06-15T09:55:16.253574Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_pa+of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T05:47:48.844990Z",
     "start_time": "2023-06-15T04:29:42.752983Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 162/162 [1:18:06<00:00, 28.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         uer/chinese_roberta_L-8_H-256\n",
      "    Label:       0.2250 0.0007\n",
      "    AV magnitude\n",
      "        all:     0.2250 0.0013\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T09:11:30.981146Z",
     "start_time": "2023-06-15T07:48:01.500200Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████| 162/162 [1:23:29<00:00, 30.92s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         clue/roberta_chinese_clue_large\n",
      "    Label:       0.2734 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.2734 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T09:29:27.932431Z",
     "start_time": "2023-06-15T09:11:30.992545Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [17:56<00:00,  6.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         ZhuiyiTechnology/T5-small\n",
      "    Label:       0.4768 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4768 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T09:47:35.889167Z",
     "start_time": "2023-06-15T09:29:27.942037Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [18:07<00:00,  6.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         clue/roberta_chinese_clue_large\n",
      "    Label:       0.2502 0.0198\n",
      "    AV magnitude\n",
      "        all:     -0.2502 0.0396\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T10:14:49.512303Z",
     "start_time": "2023-06-15T09:57:10.150277Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 162/162 [17:39<00:00,  6.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         hfl/chinese-electra-small-generator\n",
      "    Label:       0.4545 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4545 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T10:32:15.692471Z",
     "start_time": "2023-06-15T10:14:49.517977Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:26<00:00,  6.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         ZhuiyiTechnology/GPT-LCCC-nezha\n",
      "    Label:       0.4209 0.0002\n",
      "    AV magnitude\n",
      "        all:     -0.4209 0.0004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T10:48:36.819478Z",
     "start_time": "2023-06-15T10:32:15.698504Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:21<00:00,  6.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         hfl/chinese-macbert-base\n",
      "    Label:       0.4654 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4654 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T11:05:22.377518Z",
     "start_time": "2023-06-15T10:48:36.824714Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:45<00:00,  6.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         Yaxin/ernie_1.0_skep_large_ch\n",
      "    Label:       0.4306 0.0002\n",
      "    AV magnitude\n",
      "        all:     -0.4306 0.0003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd', model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T11:05:22.384404Z",
     "start_time": "2023-06-15T11:05:22.382576Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_pa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T12:22:53.913494Z",
     "start_time": "2023-06-15T11:05:22.393255Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:17:31<00:00, 28.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         uer/chinese_roberta_L-10_H-256\n",
      "    Label:       0.2878 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.2878 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T13:42:50.071954Z",
     "start_time": "2023-06-15T12:22:53.918857Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:19:56<00:00, 29.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         ZhuiyiTechnology/SimBERT-tiny\n",
      "    Label:       0.2644 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.2644 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T13:59:44.545728Z",
     "start_time": "2023-06-15T13:42:50.077470Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:54<00:00,  6.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         hfl/cino-large\n",
      "    Label:       0.3939 0.0004\n",
      "    AV magnitude\n",
      "        all:     0.3939 0.0009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T14:16:56.023777Z",
     "start_time": "2023-06-15T13:59:44.551011Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:11<00:00,  6.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         voidful/albert_chinese_xxlarge\n",
      "    Label:       0.3382 0.0024\n",
      "    AV magnitude\n",
      "        all:     -0.3382 0.0048\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T14:33:48.329691Z",
     "start_time": "2023-06-15T14:16:56.029167Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:52<00:00,  6.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         hfl/chinese-electra-base-discriminator\n",
      "    Label:       0.4739 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4739 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T14:50:58.196410Z",
     "start_time": "2023-06-15T14:33:48.334869Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:09<00:00,  6.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         hfl/chinese-electra-180g-small-generator\n",
      "    Label:       0.4345 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4345 0.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T15:07:23.282779Z",
     "start_time": "2023-06-15T14:50:58.201644Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:25<00:00,  6.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         clue/roberta_chinese_3L312_clue_tiny\n",
      "    Label:       0.4570 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.4570 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T15:24:02.509763Z",
     "start_time": "2023-06-15T15:07:23.288456Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:39<00:00,  6.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         voidful/dpr-ctx_encoder-bert-base-multilingual\n",
      "    Label:       0.3414 0.0027\n",
      "    AV magnitude\n",
      "        all:     -0.3414 0.0054\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd', model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T15:24:02.516093Z",
     "start_time": "2023-06-15T15:24:02.514457Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_na+of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T16:41:28.096341Z",
     "start_time": "2023-06-15T15:24:02.524088Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:17:25<00:00, 28.68s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         clue/roberta_chinese_clue_large\n",
      "    Label:       0.4178 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4178 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T18:01:05.662515Z",
     "start_time": "2023-06-15T16:41:28.101903Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:19:37<00:00, 29.49s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         clue/roberta_chinese_base\n",
      "    Label:       nan nan\n",
      "    AV magnitude\n",
      "        all:     nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T18:17:58.818693Z",
     "start_time": "2023-06-15T18:01:05.667666Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:53<00:00,  6.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         clue/xlnet_chinese_large\n",
      "    Label:       0.3484 0.0018\n",
      "    AV magnitude\n",
      "        all:     0.3484 0.0036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T18:35:10.122662Z",
     "start_time": "2023-06-15T18:17:58.823808Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:11<00:00,  6.37s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         uer/chinese_roberta_L-10_H-128\n",
      "    Label:       0.2938 0.0075\n",
      "    AV magnitude\n",
      "        all:     -0.2938 0.0150\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T18:51:57.803574Z",
     "start_time": "2023-06-15T18:35:10.128212Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:47<00:00,  6.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         clue/roberta_chinese_base\n",
      "    Label:       nan nan\n",
      "    AV magnitude\n",
      "        all:     nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T19:09:01.759379Z",
     "start_time": "2023-06-15T18:51:57.808978Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:03<00:00,  6.32s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         google/BERT-base\n",
      "    Label:       0.3528 0.0016\n",
      "    AV magnitude\n",
      "        all:     -0.3528 0.0032\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T19:25:20.273273Z",
     "start_time": "2023-06-15T19:09:01.764286Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:18<00:00,  6.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         clue/roberta_chinese_clue_large\n",
      "    Label:       0.4678 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4678 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T19:41:53.393643Z",
     "start_time": "2023-06-15T19:25:20.278499Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:33<00:00,  6.13s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         hfl/chinese-bert-wwm\n",
      "    Label:       0.4769 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.4769 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd', model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T19:41:53.400296Z",
     "start_time": "2023-06-15T19:41:53.398681Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_na'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T20:59:08.088252Z",
     "start_time": "2023-06-15T19:41:53.405201Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:17:14<00:00, 28.61s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         clue/roberta_chinese_pair_large\n",
      "    Label:       0.2004 0.0022\n",
      "    AV magnitude\n",
      "        all:     0.2004 0.0043\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T22:19:01.177755Z",
     "start_time": "2023-06-15T20:59:08.093528Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:19:53<00:00, 29.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         ZhuiyiTechnology/RoFormer-GPT\n",
      "    Label:       0.1954 0.0027\n",
      "    AV magnitude\n",
      "        all:     -0.1954 0.0054\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T22:35:53.528652Z",
     "start_time": "2023-06-15T22:19:01.183287Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:52<00:00,  6.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         clue/roberta_chinese_clue_tiny\n",
      "    Label:       0.1115 0.1827\n",
      "    AV magnitude\n",
      "        all:     0.1115 0.3654\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T22:53:02.041318Z",
     "start_time": "2023-06-15T22:35:53.534013Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:08<00:00,  6.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         uer/gpt2-chinese-ancient\n",
      "    Label:       0.1842 0.0663\n",
      "    AV magnitude\n",
      "        all:     -0.1842 0.1326\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T23:09:53.400816Z",
     "start_time": "2023-06-15T22:53:02.046593Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:51<00:00,  6.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         hfl/chinese-electra-small-ex-generator\n",
      "    Label:       0.1577 0.0995\n",
      "    AV magnitude\n",
      "        all:     0.1577 0.1990\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T23:27:01.489662Z",
     "start_time": "2023-06-15T23:09:53.406131Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:08<00:00,  6.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         clue/roberta_chinese_large\n",
      "    Label:       0.2636 0.0149\n",
      "    AV magnitude\n",
      "        all:     -0.2636 0.0298\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-15T23:43:38.066636Z",
     "start_time": "2023-06-15T23:27:01.494698Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:36<00:00,  6.15s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         clue/roberta_chinese_pair_large\n",
      "    Label:       nan nan\n",
      "    AV magnitude\n",
      "        all:     nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T00:00:43.379447Z",
     "start_time": "2023-06-15T23:43:38.072235Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:05<00:00,  6.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         hfl/chinese-legal-electra-small-generator\n",
      "    Label:       0.4429 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4429 0.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd', model)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T00:00:43.387524Z",
     "start_time": "2023-06-16T00:00:43.385698Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T01:18:11.673642Z",
     "start_time": "2023-06-16T00:00:43.394404Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:17:28<00:00, 28.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         uer/t5-v1_1-small-chinese-cluecorpussmall\n",
      "    Label:       0.4641 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4641 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T02:37:53.564803Z",
     "start_time": "2023-06-16T01:18:11.679129Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████| 162/162 [1:19:41<00:00, 29.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         uer/roberta-base-finetuned-dianping-chinese\n",
      "    Label:       0.2304 0.0005\n",
      "    AV magnitude\n",
      "        all:     -0.2304 0.0010\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T02:54:47.114540Z",
     "start_time": "2023-06-16T02:37:53.571754Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:53<00:00,  6.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         ZhuiyiTechnology/SimBERT-small\n",
      "    Label:       0.3738 0.0008\n",
      "    AV magnitude\n",
      "        all:     0.3738 0.0017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T03:11:57.771224Z",
     "start_time": "2023-06-16T02:54:47.120329Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:10<00:00,  6.36s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         uer/chinese_roberta_L-6_H-256\n",
      "    Label:       0.3616 0.0012\n",
      "    AV magnitude\n",
      "        all:     -0.3616 0.0024\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T03:28:52.229719Z",
     "start_time": "2023-06-16T03:11:57.776676Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:54<00:00,  6.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         uer/chinese_roberta_L-2_H-512\n",
      "    Label:       0.3578 0.0014\n",
      "    AV magnitude\n",
      "        all:     0.3578 0.0027\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T03:45:59.436926Z",
     "start_time": "2023-06-16T03:28:52.235203Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [17:07<00:00,  6.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         uer/roberta-base-finetuned-dianping-chinese\n",
      "    Label:       0.2852 0.0092\n",
      "    AV magnitude\n",
      "        all:     -0.2852 0.0184\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T04:02:21.545159Z",
     "start_time": "2023-06-16T03:45:59.442349Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:22<00:00,  6.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         uer/t5-v1_1-small-chinese-cluecorpussmall\n",
      "    Label:       0.5422 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.5422 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd', model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-16T04:18:59.698508Z",
     "start_time": "2023-06-16T04:02:21.550637Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████| 162/162 [16:38<00:00,  6.16s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         uer/gpt2-chinese-ancient\n",
      "    Label:       0.3485 0.0022\n",
      "    AV magnitude\n",
      "        all:     -0.3485 0.0044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd', model)"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}