tbl1_SDT-AT_PLM-wv.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "toc": true
   },
   "source": [
    "<h1>Table of Contents<span class=\"tocSkip\"></span></h1>\n",
    "<div class=\"toc\"><ul class=\"toc-item\"><li><span><a href=\"#AA+OF\" data-toc-modified-id=\"AA+OF-1\"><span class=\"toc-item-num\">1&nbsp;&nbsp;</span>AA+OF</a></span></li><li><span><a href=\"#AA\" data-toc-modified-id=\"AA-2\"><span class=\"toc-item-num\">2&nbsp;&nbsp;</span>AA</a></span></li><li><span><a href=\"#PA+OF\" data-toc-modified-id=\"PA+OF-3\"><span class=\"toc-item-num\">3&nbsp;&nbsp;</span>PA+OF</a></span></li><li><span><a href=\"#PA\" data-toc-modified-id=\"PA-4\"><span class=\"toc-item-num\">4&nbsp;&nbsp;</span>PA</a></span></li><li><span><a href=\"#NA+OF\" data-toc-modified-id=\"NA+OF-5\"><span class=\"toc-item-num\">5&nbsp;&nbsp;</span>NA+OF</a></span></li><li><span><a href=\"#NA\" data-toc-modified-id=\"NA-6\"><span class=\"toc-item-num\">6&nbsp;&nbsp;</span>NA</a></span></li><li><span><a href=\"#OF\" data-toc-modified-id=\"OF-7\"><span class=\"toc-item-num\">7&nbsp;&nbsp;</span>OF</a></span></li></ul></div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T23:59:15.353399Z",
     "start_time": "2023-06-13T23:59:14.484283Z"
    }
   },
   "outputs": [],
   "source": [
    "from sdt import *\n",
    "from evaluate import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T12:33:18.979854Z",
     "start_time": "2023-06-13T12:33:18.975774Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_aa+of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T13:26:00.893063Z",
     "start_time": "2023-06-13T12:33:40.541042Z"
    },
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [52:20<00:00, 26.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word+char+ngram_zhihu_m\n",
      "    Label:       0.2280 0.0006\n",
      "    AV magnitude\n",
      "        all:     0.2280 0.0011\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T14:18:27.858391Z",
     "start_time": "2023-06-13T13:26:00.902573Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [52:26<00:00, 26.22s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/cw_word_c\n",
      "    Label:       0.2704 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.2704 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T14:29:53.787380Z",
     "start_time": "2023-06-13T14:18:27.867242Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:25<00:00,  5.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+char+ngram_wiki_c\n",
      "    Label:       0.4511 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.4511 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T14:41:29.904236Z",
     "start_time": "2023-06-13T14:29:53.797411Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:36<00:00,  5.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         tencent/v020-d100-small_c\n",
      "    Label:       0.2910 0.0080\n",
      "    AV magnitude\n",
      "        all:     -0.2910 0.0161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T14:52:54.723526Z",
     "start_time": "2023-06-13T14:41:29.912698Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:24<00:00,  5.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word+char_zhihu_m\n",
      "    Label:       0.4439 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.4439 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T15:04:39.742803Z",
     "start_time": "2023-06-13T14:52:54.732259Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:45<00:00,  5.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word+char+ngram_baidu_m\n",
      "    Label:       0.4569 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.4569 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T15:15:55.899049Z",
     "start_time": "2023-06-13T15:04:39.751619Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:16<00:00,  5.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/word+ngram_literature_m\n",
      "    Label:       0.4427 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.4427 0.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T15:27:14.620619Z",
     "start_time": "2023-06-13T15:15:55.911334Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:18<00:00,  5.66s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/cw_char_1_c\n",
      "    Label:       0.4861 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.4861 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# AA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T15:27:14.632835Z",
     "start_time": "2023-06-13T15:27:14.631208Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_aa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T16:18:10.442839Z",
     "start_time": "2023-06-13T15:27:14.646462Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [50:55<00:00, 25.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/tw_ngram_1_m\n",
      "    Label:       0.2452 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.2452 0.0005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T17:10:34.953054Z",
     "start_time": "2023-06-13T16:18:10.453148Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [52:24<00:00, 26.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word_literature_m\n",
      "    Label:       0.2302 0.0005\n",
      "    AV magnitude\n",
      "        all:     -0.2302 0.0010\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T17:22:02.024576Z",
     "start_time": "2023-06-13T17:10:34.963151Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:27<00:00,  5.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+char_financial_c\n",
      "    Label:       0.4152 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.4152 0.0004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T17:33:39.935775Z",
     "start_time": "2023-06-13T17:22:02.034898Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:37<00:00,  5.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         tencent/v010-d200_m\n",
      "    Label:       0.2642 0.0147\n",
      "    AV magnitude\n",
      "        all:     -0.2642 0.0294\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T17:45:05.786207Z",
     "start_time": "2023-06-13T17:33:39.945887Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:25<00:00,  5.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word_wiki_m\n",
      "    Label:       0.4195 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.4195 0.0004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T17:56:43.029530Z",
     "start_time": "2023-06-13T17:45:05.797519Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:37<00:00,  5.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word+char+ngram_sogou_m\n",
      "    Label:       0.3744 0.0008\n",
      "    AV magnitude\n",
      "        all:     -0.3744 0.0017\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T18:07:50.559459Z",
     "start_time": "2023-06-13T17:56:43.041210Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:07<00:00,  5.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/tw_ngram_1_m\n",
      "    Label:       0.4149 0.0003\n",
      "    AV magnitude\n",
      "        all:     0.4149 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T18:19:08.692556Z",
     "start_time": "2023-06-13T18:07:50.570408Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:18<00:00,  5.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/word+ngram_wiki_c\n",
      "    Label:       0.4556 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4556 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T18:19:08.705248Z",
     "start_time": "2023-06-13T18:19:08.703122Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_pa+of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T19:09:54.428344Z",
     "start_time": "2023-06-13T18:19:08.714732Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [50:45<00:00, 25.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/tw_char_2_c\n",
      "    Label:       0.2447 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.2447 0.0005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T20:02:17.778717Z",
     "start_time": "2023-06-13T19:09:54.438538Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [52:23<00:00, 26.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word+char+ngram_financial_c\n",
      "    Label:       0.2357 0.0004\n",
      "    AV magnitude\n",
      "        all:     -0.2357 0.0008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T20:13:43.824601Z",
     "start_time": "2023-06-13T20:02:17.798471Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:26<00:00,  5.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+char_zhihu_c\n",
      "    Label:       0.4092 0.0003\n",
      "    AV magnitude\n",
      "        all:     0.4092 0.0005\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T20:25:21.820322Z",
     "start_time": "2023-06-13T20:13:43.834868Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:37<00:00,  5.82s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word_literature_m\n",
      "    Label:       0.2710 0.0127\n",
      "    AV magnitude\n",
      "        all:     -0.2710 0.0254\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T20:36:47.182151Z",
     "start_time": "2023-06-13T20:25:21.831002Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:25<00:00,  5.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/tw_ngram_2_m\n",
      "    Label:       0.4402 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.4402 0.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T20:48:23.415828Z",
     "start_time": "2023-06-13T20:36:47.192185Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:36<00:00,  5.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/tw_char_2_c\n",
      "    Label:       0.4256 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4256 0.0003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T20:59:31.143439Z",
     "start_time": "2023-06-13T20:48:23.426787Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:07<00:00,  5.56s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/word+ngram_literature_m\n",
      "    Label:       0.4624 0.0001\n",
      "    AV magnitude\n",
      "        all:     0.4624 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T21:10:47.456597Z",
     "start_time": "2023-06-13T20:59:31.153721Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:16<00:00,  5.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/tw_position_1_c\n",
      "    Label:       0.3838 0.0008\n",
      "    AV magnitude\n",
      "        all:     -0.3838 0.0016\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# PA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T23:59:27.042300Z",
     "start_time": "2023-06-13T23:59:27.034400Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_pa'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T22:01:37.611913Z",
     "start_time": "2023-06-13T21:10:47.490383Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [50:50<00:00, 25.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word+char+ngram_wiki_c\n",
      "    Label:       0.2265 0.0006\n",
      "    AV magnitude\n",
      "        all:     0.2265 0.0012\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T22:53:58.538640Z",
     "start_time": "2023-06-13T22:01:37.622627Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [52:20<00:00, 26.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/tw_char_3_m\n",
      "    Label:       0.2331 0.0004\n",
      "    AV magnitude\n",
      "        all:     -0.2331 0.0009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T23:05:23.879557Z",
     "start_time": "2023-06-13T22:53:58.548801Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:25<00:00,  5.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+char_c\n",
      "    Label:       0.3939 0.0004\n",
      "    AV magnitude\n",
      "        all:     0.3939 0.0009\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T23:16:59.624290Z",
     "start_time": "2023-06-13T23:05:23.889183Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:35<00:00,  5.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/tw_char_1_c\n",
      "    Label:       0.2232 0.0337\n",
      "    AV magnitude\n",
      "        all:     -0.2232 0.0673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T23:28:25.152368Z",
     "start_time": "2023-06-13T23:16:59.634743Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:25<00:00,  5.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/cw_ngram_3_c\n",
      "    Label:       0.4635 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.4635 0.0001\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T23:40:02.199216Z",
     "start_time": "2023-06-13T23:28:25.163588Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:37<00:00,  5.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/tw_position_2_m\n",
      "    Label:       0.4164 0.0002\n",
      "    AV magnitude\n",
      "        all:     -0.4164 0.0004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-13T23:51:08.045810Z",
     "start_time": "2023-06-13T23:40:02.209900Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:05<00:00,  5.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/cw_word_c\n",
      "    Label:       0.4322 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.4322 0.0003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T00:10:46.100077Z",
     "start_time": "2023-06-13T23:59:29.836080Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:16<00:00,  5.64s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/word+char_financial_m\n",
      "    Label:       0.3524 0.0020\n",
      "    AV magnitude\n",
      "        all:     -0.3524 0.0040\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NA+OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T00:10:46.117870Z",
     "start_time": "2023-06-14T00:10:46.115804Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_na+of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T01:01:25.981799Z",
     "start_time": "2023-06-14T00:10:46.132318Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [50:39<00:00, 25.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word+char+ngram_c\n",
      "    Label:       nan nan\n",
      "    AV magnitude\n",
      "        all:     nan nan\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T01:53:39.852853Z",
     "start_time": "2023-06-14T01:01:25.999022Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [52:13<00:00, 26.12s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word+char+ngram_literature_c\n",
      "    Label:       0.2866 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.2866 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T02:05:05.384602Z",
     "start_time": "2023-06-14T01:53:39.867509Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:25<00:00,  5.71s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+char+ngram_financial_m\n",
      "    Label:       0.4064 0.0003\n",
      "    AV magnitude\n",
      "        all:     0.4064 0.0006\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T02:16:40.272955Z",
     "start_time": "2023-06-14T02:05:05.399384Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:34<00:00,  5.79s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+ngram_literature_m\n",
      "    Label:       0.3715 0.0009\n",
      "    AV magnitude\n",
      "        all:     -0.3715 0.0018\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T02:28:03.820447Z",
     "start_time": "2023-06-14T02:16:40.287823Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:23<00:00,  5.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word_weibo_c\n",
      "    Label:       0.2415 0.0236\n",
      "    AV magnitude\n",
      "        all:     0.2415 0.0473\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T02:39:37.030246Z",
     "start_time": "2023-06-14T02:28:03.839961Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:33<00:00,  5.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/tw_position_2_c\n",
      "    Label:       0.3167 0.0043\n",
      "    AV magnitude\n",
      "        all:     -0.3167 0.0085\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T02:50:42.038003Z",
     "start_time": "2023-06-14T02:39:37.043961Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:04<00:00,  5.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         tencent/v020-d200-large_m\n",
      "    Label:       0.4050 0.0004\n",
      "    AV magnitude\n",
      "        all:     0.4050 0.0008\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T03:01:56.354910Z",
     "start_time": "2023-06-14T02:50:42.053703Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:14<00:00,  5.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/word+ngram_wiki_c\n",
      "    Label:       0.4419 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4419 0.0002\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NA"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T03:01:56.370542Z",
     "start_time": "2023-06-14T03:01:56.368684Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_na'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T03:52:50.617972Z",
     "start_time": "2023-06-14T03:01:56.385256Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [50:54<00:00, 25.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word+ngram_weibo_c\n",
      "    Label:       0.1779 0.0058\n",
      "    AV magnitude\n",
      "        all:     0.1779 0.0115\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T04:46:17.677447Z",
     "start_time": "2023-06-14T03:52:50.631757Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [53:27<00:00, 26.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/tw_ngram_3_m\n",
      "    Label:       0.1871 0.0039\n",
      "    AV magnitude\n",
      "        all:     -0.1871 0.0078\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T04:57:48.419039Z",
     "start_time": "2023-06-14T04:46:17.692195Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:30<00:00,  5.76s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+char_financial_m\n",
      "    Label:       0.1359 0.1346\n",
      "    AV magnitude\n",
      "        all:     0.1359 0.2691\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T05:09:30.570854Z",
     "start_time": "2023-06-14T04:57:48.436040Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:42<00:00,  5.85s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word+ngram_wiki_c\n",
      "    Label:       0.1118 0.1820\n",
      "    AV magnitude\n",
      "        all:     -0.1118 0.3641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T05:21:00.334533Z",
     "start_time": "2023-06-14T05:09:30.588626Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:29<00:00,  5.75s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word+char+ngram_baidu_c\n",
      "    Label:       0.1577 0.0995\n",
      "    AV magnitude\n",
      "        all:     0.1577 0.1990\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T05:32:40.482398Z",
     "start_time": "2023-06-14T05:21:00.358104Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:40<00:00,  5.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word_sogou_m\n",
      "    Label:       0.1703 0.0825\n",
      "    AV magnitude\n",
      "        all:     -0.1703 0.1649\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T05:43:51.044471Z",
     "start_time": "2023-06-14T05:32:40.498177Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:10<00:00,  5.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/tw_char_1_m\n",
      "    Label:       0.4256 0.0002\n",
      "    AV magnitude\n",
      "        all:     0.4256 0.0004\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T05:55:11.472931Z",
     "start_time": "2023-06-14T05:43:51.058482Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:20<00:00,  5.67s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/cw_position_1_c\n",
      "    Label:       0.3559 0.0018\n",
      "    AV magnitude\n",
      "        all:     -0.3559 0.0036\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# OF"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T05:55:11.490480Z",
     "start_time": "2023-06-14T05:55:11.488598Z"
    }
   },
   "outputs": [],
   "source": [
    "mode_name = 'plm_of'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T06:45:57.507060Z",
     "start_time": "2023-06-14T05:55:11.505129Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [50:45<00:00, 25.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/word+char+ngram_m\n",
      "    Label:       0.5093 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.5093 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T07:38:05.945966Z",
     "start_time": "2023-06-14T06:45:57.531599Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [52:08<00:00, 26.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - all\n",
      "SPEARMAN         sgns/tw_ngram_1_c\n",
      "    Label:       0.2098 0.0014\n",
      "    AV magnitude\n",
      "        all:     -0.2098 0.0028\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', 'all')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T07:49:32.549983Z",
     "start_time": "2023-06-14T07:38:05.961089Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:26<00:00,  5.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/tw_char_2_c\n",
      "    Label:       0.2835 0.0096\n",
      "    AV magnitude\n",
      "        all:     0.2835 0.0192\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T08:01:09.338509Z",
     "start_time": "2023-06-14T07:49:32.565142Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:36<00:00,  5.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 1st\n",
      "SPEARMAN         sgns/word_literature_c\n",
      "    Label:       0.3030 0.0060\n",
      "    AV magnitude\n",
      "        all:     -0.3030 0.0120\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '1st')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T08:12:35.736539Z",
     "start_time": "2023-06-14T08:01:09.353349Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:26<00:00,  5.72s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         sgns/word+ngram_sogou_m\n",
      "    Label:       0.2320 0.0284\n",
      "    AV magnitude\n",
      "        all:     0.2320 0.0569\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T08:24:11.173854Z",
     "start_time": "2023-06-14T08:12:35.751480Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:35<00:00,  5.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 2nd\n",
      "SPEARMAN         tencent/v010-d200_m\n",
      "    Label:       0.4276 0.0001\n",
      "    AV magnitude\n",
      "        all:     -0.4276 0.0003\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '2nd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T08:35:16.515472Z",
     "start_time": "2023-06-14T08:24:11.189814Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:05<00:00,  5.54s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         sgns/word+ngram_sikuquanshu\n",
      "    Label:       0.5615 0.0000\n",
      "    AV magnitude\n",
      "        all:     0.5615 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '1', '3rd')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2023-06-14T08:46:31.606997Z",
     "start_time": "2023-06-14T08:35:16.530651Z"
    },
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 120/120 [11:15<00:00,  5.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Nested LOOCV - 3rd\n",
      "SPEARMAN         tencent/v020-d200-small_c\n",
      "    Label:       0.5087 0.0000\n",
      "    AV magnitude\n",
      "        all:     -0.5087 0.0000\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "final_eval(mode_name, '2', '3rd')"
   ]
  }
 ],
 "metadata": {
  "hide_input": false,
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.10"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autoclose": false,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": true,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "165px"
   },
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}