_CLF_MLP_FINAL_CV copy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Preparation"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Colab"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "dQSdijCsmtPH",
    "outputId": "c9932f69-53b1-4d31-fb03-784b29bf35f7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: transformers in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (4.3.3)\n",
      "Requirement already satisfied: numpy>=1.17 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (1.20.1)\n",
      "Requirement already satisfied: regex!=2019.12.17 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (2020.11.13)\n",
      "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (0.10.1)\n",
      "Requirement already satisfied: packaging in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (20.9)\n",
      "Requirement already satisfied: sacremoses in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (0.0.43)\n",
      "Requirement already satisfied: tqdm>=4.27 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (4.58.0)\n",
      "Requirement already satisfied: filelock in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (3.0.12)\n",
      "Requirement already satisfied: requests in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from transformers) (2.25.1)\n",
      "Requirement already satisfied: pyparsing>=2.0.2 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from packaging->transformers) (2.4.7)\n",
      "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from requests->transformers) (1.26.3)\n",
      "Requirement already satisfied: idna<3,>=2.5 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from requests->transformers) (2.10)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from requests->transformers) (2020.12.5)\n",
      "Requirement already satisfied: chardet<5,>=3.0.2 in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from requests->transformers) (4.0.0)\n",
      "Requirement already satisfied: click in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from sacremoses->transformers) (7.1.2)\n",
      "Requirement already satisfied: joblib in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from sacremoses->transformers) (1.0.1)\n",
      "Requirement already satisfied: six in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from sacremoses->transformers) (1.15.0)\n",
      "Requirement already satisfied: torch in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (1.8.0)\n",
      "Requirement already satisfied: typing_extensions in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from torch) (3.7.4.3)\n",
      "Requirement already satisfied: numpy in /Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages (from torch) (1.20.1)\n",
      "^C\n",
      "Traceback (most recent call last):\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/bin/pip\", line 11, in <module>\n",
      "    sys.exit(main())\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_internal/cli/main.py\", line 71, in main\n",
      "    command = create_command(cmd_name, isolated=(\"--isolated\" in cmd_args))\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_internal/commands/__init__.py\", line 96, in create_command\n",
      "    module = importlib.import_module(module_path)\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/importlib/__init__.py\", line 127, in import_module\n",
      "    return _bootstrap._gcd_import(name[level:], package, level)\n",
      "  File \"<frozen importlib._bootstrap>\", line 1014, in _gcd_import\n",
      "  File \"<frozen importlib._bootstrap>\", line 991, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 975, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 671, in _load_unlocked\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 783, in exec_module\n",
      "  File \"<frozen importlib._bootstrap>\", line 219, in _call_with_frames_removed\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_internal/commands/install.py\", line 15, in <module>\n",
      "    from pip._internal.cli.req_command import RequirementCommand, with_cleanup\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_internal/cli/req_command.py\", line 16, in <module>\n",
      "    from pip._internal.index.collector import LinkCollector\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_internal/index/collector.py\", line 16, in <module>\n",
      "    from pip._vendor import html5lib, requests\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_vendor/requests/__init__.py\", line 44, in <module>\n",
      "    from pip._vendor import chardet\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_vendor/chardet/__init__.py\", line 19, in <module>\n",
      "    from .universaldetector import UniversalDetector\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_vendor/chardet/universaldetector.py\", line 47, in <module>\n",
      "    from .mbcsgroupprober import MBCSGroupProber\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_vendor/chardet/mbcsgroupprober.py\", line 32, in <module>\n",
      "    from .sjisprober import SJISProber\n",
      "  File \"/Users/june/miniconda3/envs/rosetta/lib/python3.8/site-packages/pip/_vendor/chardet/sjisprober.py\", line 31, in <module>\n",
      "    from .jpcntx import SJISContextAnalysis\n",
      "  File \"<frozen importlib._bootstrap>\", line 991, in _find_and_load\n",
      "  File \"<frozen importlib._bootstrap>\", line 975, in _find_and_load_unlocked\n",
      "  File \"<frozen importlib._bootstrap>\", line 671, in _load_unlocked\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 779, in exec_module\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 874, in get_code\n",
      "  File \"<frozen importlib._bootstrap_external>\", line 973, in get_data\n",
      "KeyboardInterrupt\n"
     ]
    }
   ],
   "source": [
    "!pip install transformers\n",
    "!pip install torch\n",
    "!pip install emoji"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "4FaLWxNckcCp",
    "outputId": "4196fd01-160b-492f-8a1e-2d53cb73aad7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/Users/june/Documents/Code/FYP/code_data\n"
     ]
    },
    {
     "ename": "ModuleNotFoundError",
     "evalue": "No module named 'google.colab'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-3-1836b0b99c66>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;31m# 실행시 등장하는 URL을 클릭하여 허용해주면 인증KEY가 나타난다. 복사하여 URL아래 빈칸에 붙여넣으면 마운트에 성공하게된다.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0;32mfrom\u001b[0m \u001b[0mgoogle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolab\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mdrive\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./MyDrive'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'google.colab'"
     ]
    }
   ],
   "source": [
    "import os\n",
    "print(os.getcwd())\n",
    "\n",
    "# 실행시 등장하는 URL을 클릭하여 허용해주면 인증KEY가 나타난다. 복사하여 URL아래 빈칸에 붙여넣으면 마운트에 성공하게된다.\n",
    "from google.colab import drive\n",
    "drive.mount('./MyDrive')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "NTiePp9YlA8R",
    "outputId": "525a66f7-82ae-4025-ac10-8b45c481c34b"
   },
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "/content/MyDrive/MyDrive/Capstone/code_data\n"
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "cd MyDrive/MyDrive/Capstone/code_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "TAXZNl6bmM7J",
    "outputId": "d2fcf6e1-a3b5-4689-9c5a-dde102b68239"
   },
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "/content/MyDrive/My Drive/Capstone/code_data\n"
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "!pwd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Library"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
    "from torch.utils.data.sampler import WeightedRandomSampler\n",
    "from torch.optim import lr_scheduler\n",
    "\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "from sklearn.metrics import accuracy_score\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import f1_score, precision_score, recall_score\n",
    "from transformers import AdamW, get_linear_schedule_with_warmup\n",
    "from sklearn.inspection import permutation_importance\n",
    "\n",
    "from fetchData import fetchdata, cv_events\n",
    "import __MLP\n",
    "# from __MLP import getSamplers, convert_df_to_unsqueezed_tensor, train_sequential, clf_report\n",
    "import random\n",
    "\n",
    "import __Preprocessing\n",
    "from sklearn.ensemble import ExtraTreesClassifier, RandomForestClassifier\n",
    "\n",
    "pd.set_option('display.max_columns', None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No GPU available, using the CPU instead.\n"
     ]
    }
   ],
   "source": [
    "if torch.cuda.is_available():       \n",
    "    device = torch.device(\"cuda\")\n",
    "    print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
    "    print('Device name:', torch.cuda.get_device_name(0))\n",
    "\n",
    "else:\n",
    "    print('No GPU available, using the CPU instead.')\n",
    "    device = torch.device(\"cpu\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "seed = 42\n",
    "if torch.cuda.is_available():\n",
    "    torch.cuda.manual_seed_all(seed_value)\n",
    "\n",
    "\n",
    "torch.manual_seed(42)\n",
    "torch.cuda.manual_seed(42)\n",
    "np.random.seed(42)\n",
    "random.seed(42)\n",
    "torch.backends.cudnn.deterministic=True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "def _init_fn(worker_id):\n",
    "    np.random.seed(int(seed))\n",
    "\n",
    "def test_data_process(X_test, y_test):\n",
    "    tensor_x1 = torch.Tensor(X_test.values).unsqueeze(1)\n",
    "    tensor_y1 = torch.Tensor(y_test.values).unsqueeze(1)\n",
    "    test_dataset = TensorDataset(tensor_x1,tensor_y1)\n",
    "\n",
    "    batch_size = 8\n",
    "\n",
    "    # train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)\n",
    "    test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n",
    "\n",
    "    data = next(iter(test_dataloader))\n",
    "    print(\"mean: %s, std: %s\" %(data[0].mean(), data[0].std()))\n",
    "\n",
    "    test_size = int(tensor_y1.size(0))\n",
    "\n",
    "    print(\"Test Size\",test_size)\n",
    "\n",
    "    # predict_batch\n",
    "    return test_dataloader, test_size\n",
    "\n",
    "\n",
    "def test_data_process(X_test, y_test):\n",
    "\n",
    "    tensor_x1 = torch.Tensor(X_test.values).unsqueeze(1)\n",
    "    tensor_y1 = torch.Tensor(y_test.values).unsqueeze(1)\n",
    "    test_dataset = TensorDataset(tensor_x1,tensor_y1)\n",
    "\n",
    "    batch_size = 8\n",
    "\n",
    "    # train_sampler, test_sampler = __MLP.getSamplers(pheme_y, tensor_x2)\n",
    "    test_dataloader = DataLoader(test_dataset, batch_size=batch_size, shuffle=False, num_workers=2)\n",
    "\n",
    "    data = next(iter(test_dataloader))\n",
    "    print(\"mean: %s, std: %s\" %(data[0].mean(), data[0].std()))\n",
    "\n",
    "    test_size = int(tensor_y1.size(0))\n",
    "\n",
    "    print(\"Test Size\",test_size)\n",
    "\n",
    "    # predict_batch\n",
    "    return test_dataloader, test_size\n",
    "\n",
    "def predict(model, criterion, val_dataloader, val_size):\n",
    "    model.eval()\n",
    "    val_label_list = []\n",
    "    # val_preds_list = []\n",
    "    running_val_preds = []\n",
    "\n",
    "    with torch.no_grad():\n",
    "        val_loss = 0.0\n",
    "        val_corrects = 0\n",
    "        f1_running = 0\n",
    "        for j, val in enumerate(val_dataloader, 0):\n",
    "            val_x, val_label = val\n",
    "            val_x, val_label = val_x.float(), val_label.float()\n",
    "            val_outputs = model(val_x)\n",
    "            val_preds = val_outputs.squeeze(1) > 0.0\n",
    "            f1_running += (f1_score(val_label, val_preds,zero_division=True) * val_x.size(0))\n",
    "            v_loss = criterion(val_outputs, val_label.unsqueeze(1))\n",
    "            val_loss += (v_loss.item() * val_x.size(0))\n",
    "            val_corrects += torch.sum(val_preds == val_label)\n",
    "            val_label_list.append(val_label)\n",
    "            running_val_preds.append(val_preds)\n",
    "\n",
    "    running_val_preds = torch.cat(running_val_preds, 0)\n",
    "    val_label_list = torch.cat(val_label_list, 0)\n",
    "    val_corrects = val_corrects\n",
    "    val_loss = val_loss/val_size\n",
    "    val_acc = val_corrects.double().numpy() / val_size\n",
    "    f1_running /= val_size\n",
    "    print(\"accuracy_score:\\t\\t%.4f\" % val_acc)\n",
    "    print('Precision Score:\\t%.4f' % precision_score(val_label_list,running_val_preds))\n",
    "    print('Recall Score:\\t\\t%.4f' % recall_score(val_label_list,running_val_preds))\n",
    "    print(\"f1_score:\\t\\t%.4f\" % f1_running)\n",
    "    print(\"Test_loss:\\t\\t%.4f\" % val_loss)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Final\n",
    "pheme_sparse_final = pd.read_csv('./data/_PHEME_sparse.csv')\n",
    "pheme_y = pd.read_csv('./data/_PHEME_target.csv').target\n",
    "pheme_pos_final = pd.read_csv('./data/_PHEME_postags.csv')\n",
    "pheme_thread_final_avg = pd.read_csv('./data/_PHEME_thread_avg.csv')\n",
    "pheme_thread_final_std = pd.read_csv('./data/_PHEME_thread_std.csv')\n",
    "\n",
    "ext_pos_final = pd.read_csv('./data/_PHEMEext_postags.csv')\n",
    "ext_sparse_final = pd.read_csv('./data/_PHEMEext_sparse.csv')\n",
    "ext_y = pd.read_csv('./data/_PHEMEext_text.csv').target\n",
    "ext_thread_final_avg = pd.read_csv('./data/_PHEMEext_thread_avg.csv')\n",
    "ext_thread_final_std = pd.read_csv('./data/_PHEMEext_thread_std.csv')\n",
    "\n",
    "pheme_bert_simple_normal = pd.read_csv('./data/_PHEME_Bert_final_simple_nrmzd.csv')\n",
    "ext_bert_simple_normal = pd.read_csv('./data/_PHEMEext_Bert_final_simple_nrmzd.csv')\n",
    "\n",
    "pheme_bert_brackets_normal = pd.read_csv('./data/_PHEME_Bert_final_brackets_nrmzd.csv')\n",
    "ext_bert_brackets_normal = pd.read_csv('./data/_PHEMEext_Bert_final_brackets_nrmzd.csv')\n",
    "\n",
    "pheme_event = pd.read_csv('./data/_PHEME_text.csv')['Event']\n",
    "ext_event = pd.read_csv('./data/_PHEMEext_text.csv').Event\n",
    "# pheme_AVGw2v = pd.read_csv('./data/_PHEME_text_AVGw2v.csv').drop(['token'],axis=1)\n",
    "# ext_AVGw2v = pd.read_csv('./data/_PHEMEext_text_AVGw2v.csv').drop(['token'],axis=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "pheme_textonly = pd.read_csv('./data/_PHEME_textonly.csv')\n",
    "ext_textonly = pd.read_csv('./data/_PHEMEext_textonly.csv')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "pheme_AVGw2v = pd.read_csv('./data/_PHEME_text_AVGw2v_final.csv')\n",
    "ext_AVGw2v = pd.read_csv('./data/_PHEMEext_text_AVGw2v_final.csv')\n",
    "\n",
    "pheme_root = pd.concat([pheme_sparse_final, pheme_pos_final],axis=1)\n",
    "ext_root = pd.concat([ext_sparse_final, ext_pos_final],axis=1)\n",
    "\n",
    "pheme_root_thread = pd.concat([pheme_root, pheme_thread_final_avg],axis=1)\n",
    "ext_root_thread = pd.concat([ext_root, ext_thread_final_avg],axis=1)\n",
    "\n",
    "pheme_total_bert= pd.concat([pheme_root_thread, pheme_bert_simple_normal],axis=1)\n",
    "ext_total_bert = pd.concat([ext_root_thread, ext_bert_simple_normal],axis=1)\n",
    "pheme_total_w2v= pd.concat([pheme_root_thread, pheme_AVGw2v],axis=1)\n",
    "ext_total_w2v = pd.concat([ext_root_thread, ext_AVGw2v],axis=1)\n",
    "\n",
    "pheme_w2v_bert = pd.concat([pheme_AVGw2v, pheme_bert_simple_normal],axis=1)\n",
    "ext_w2v_bert = pd.concat([ext_AVGw2v, ext_bert_simple_normal],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_event = pd.concat([pheme_event,ext_event],axis=0, ignore_index=True)\n",
    "all_y = pd.concat([pheme_y,ext_y],axis=0, ignore_index=True)\n",
    "\n",
    "all_root = pd.concat([pheme_root, ext_root],axis=0, ignore_index=True)\n",
    "all_thread = pd.concat([pheme_thread_final_avg, ext_thread_final_avg],axis=0, ignore_index=True)\n",
    "all_bert_simple = pd.concat([pheme_bert_simple_normal,ext_bert_simple_normal],axis=0,ignore_index=True)\n",
    "all_AVGw2v = pd.concat([pheme_AVGw2v,ext_AVGw2v],axis=0,ignore_index=True)\n",
    "all_w2v_bert = pd.concat([pheme_w2v_bert,ext_w2v_bert],axis=0,ignore_index=True)\n",
    "all_root_thread = pd.concat([pheme_root_thread,ext_root_thread],axis=0,ignore_index=True)\n",
    "all_total_bert = pd.concat([pheme_total_bert,ext_total_bert],axis=0,ignore_index=True)\n",
    "all_total_w2v = pd.concat([pheme_total_w2v,ext_total_w2v],axis=0,ignore_index=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pheme_sparse_final (5802, 28)\n",
      "Index(['emoji_count', 'URLcount', 'has_media', 'Skepticism', 'MentionCount',\n",
      "       'FirstPersonPronoun', 'SecondPersonPronoun', 'ThirdPersonPronoun',\n",
      "       'test_auxiliary', 'test_tentat', 'test_certain', 'Numeral',\n",
      "       'char_count', 'word_count', 'HashTag', 'has_question', 'has_exclaim',\n",
      "       'has_period', 'capital_ratio', 'retweet_count', 'tweet_count',\n",
      "       'listed_count', 'friends_count', 'follower_count', 'followers/friend',\n",
      "       'favourites_count', 'account_age_days', 'verified'],\n",
      "      dtype='object') \n",
      "\n",
      "pheme_pos_final (5802, 21)\n",
      "{('n', 'pre/postposition/subordinating conjunction'), ('v', 'proper noun + possessive'), ('o', 'adverb'), ('g', 'common noun'), ('t', 'verb incl. copula, auxiliaries'), ('&', 'coordinating conjunction'), ('#', 'Hashtag'), ('p', 'nominal + possessive'), ('!', 'Interjection'), ('u', 'existential there, predeterminers'), ('d', 'nominal + verbal'), ('s', 'URL or email'), ('^', 'proper noun'), ('x', 'discourse marker'), ('r', 'verb particle'), (',', 'punctuation'), ('l', 'pronoun'), ('@', 'at-mention'), ('a', 'adjectivedeterminerother')} \n",
      "\n",
      "pheme_thread_final (5802, 52)\n",
      "Index(['depth', 'SUM FriendsCount', 'AVG FriendsCount', 'AVG WordCount',\n",
      "       'SUM WordCount', 'AVG CharCount', 'AVG HashTag', 'SUM HashTag',\n",
      "       'Ratio HashTag', 'SUM Url', 'AVG Url', 'RATIO Url', 'SUM Mention',\n",
      "       'AVG Mention', 'Ratio Mention', 'AVG Statues', 'AVG Listed',\n",
      "       'AVG Follower', 'AVG followers/friend', 'AVG favorite', 'Tweets Count',\n",
      "       'Ratio Verified', 'SUM Verified', 'SUM RT', 'AVG RT', 'AVG AccAge',\n",
      "       'thread_time', 'AVG Emoji', 'RATIO Emoji', 'Ratio Media',\n",
      "       'RATIO Question', 'RATIO Exclaim', 'RATIO Period', 'AVG FPP', 'AVG SPP',\n",
      "       'AVG TPP', 'AVG Skepticism', 'Ratio Skepticism', 'test_auxiliary',\n",
      "       'test_tentat', 'test_certain', 'root_user_ratio', 'unique_user_ratio',\n",
      "       'unique_user_sum', 'NodeThreadCount_0', 'NodeThreadCount_25',\n",
      "       'NodeThreadCount_5', 'NodeThreadCount_75', 'NodeThreadRatio_0',\n",
      "       'NodeThreadRatio_25', 'NodeThreadRatio_5', 'NodeThreadRatio_75'],\n",
      "      dtype='object') \n",
      "\n",
      "pheme_bert_simple_normal (5802, 768)\n",
      "Index(['BERTEmbed_0', 'BERTEmbed_1', 'BERTEmbed_2', 'BERTEmbed_3',\n",
      "       'BERTEmbed_4', 'BERTEmbed_5', 'BERTEmbed_6', 'BERTEmbed_7',\n",
      "       'BERTEmbed_8', 'BERTEmbed_9',\n",
      "       ...\n",
      "       'BERTEmbed_758', 'BERTEmbed_759', 'BERTEmbed_760', 'BERTEmbed_761',\n",
      "       'BERTEmbed_762', 'BERTEmbed_763', 'BERTEmbed_764', 'BERTEmbed_765',\n",
      "       'BERTEmbed_766', 'BERTEmbed_767'],\n",
      "      dtype='object', length=768) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"pheme_sparse_final\", pheme_sparse_final.shape)\n",
    "print(pheme_sparse_final.columns, \"\\n\")\n",
    "print(\"pheme_pos_final\", pheme_pos_final.shape)\n",
    "x = zip(pheme_pos_final.columns.values, ['Interjection', 'Hashtag', 'coordinating conjunction', 'punctuation', 'at-mention', 'proper noun', 'adjective' 'determiner' 'other', 'nominal + verbal',\n",
    "    'common noun', 'pronoun', 'pre/postposition/subordinating conjunction', 'adverb', 'nominal + possessive', 'verb particle', 'URL or email', 'verb incl. copula, auxiliaries',\n",
    " 'existential there, predeterminers', 'proper noun + possessive', 'discourse marker'])\n",
    "print(set(x), \"\\n\")\n",
    "print(\"pheme_thread_final\", pheme_thread_final_avg.shape)\n",
    "print(pheme_thread_final_avg.columns, \"\\n\")\n",
    "print(\"pheme_bert_simple_normal\", pheme_bert_simple_normal.shape)\n",
    "print(pheme_bert_simple_normal.columns, \"\\n\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "pheme_sparse_final (623, 28)\n",
      "Index(['emoji_count', 'URLcount', 'has_media', 'Skepticism', 'MentionCount',\n",
      "       'FirstPersonPronoun', 'SecondPersonPronoun', 'ThirdPersonPronoun',\n",
      "       'test_auxiliary', 'test_tentat', 'test_certain', 'Numeral',\n",
      "       'char_count', 'word_count', 'HashTag', 'has_question', 'has_exclaim',\n",
      "       'has_period', 'capital_ratio', 'retweet_count', 'tweet_count',\n",
      "       'listed_count', 'friends_count', 'follower_count', 'followers/friend',\n",
      "       'favourites_count', 'account_age_days', 'verified'],\n",
      "      dtype='object') \n",
      "\n",
      "pheme_pos_final (623, 21)\n",
      "{('n', 'pre/postposition/subordinating conjunction'), ('v', 'proper noun + possessive'), ('o', 'adverb'), ('g', 'common noun'), ('t', 'verb incl. copula, auxiliaries'), ('&', 'coordinating conjunction'), ('#', 'Hashtag'), ('p', 'nominal + possessive'), ('!', 'Interjection'), ('u', 'existential there, predeterminers'), ('d', 'nominal + verbal'), ('s', 'URL or email'), ('^', 'proper noun'), ('x', 'discourse marker'), ('r', 'verb particle'), (',', 'punctuation'), ('l', 'pronoun'), ('@', 'at-mention'), ('a', 'adjectivedeterminerother')} \n",
      "\n",
      "pheme_thread_final (623, 52)\n",
      "Index(['depth', 'SUM FriendsCount', 'AVG FriendsCount', 'AVG WordCount',\n",
      "       'SUM WordCount', 'AVG CharCount', 'AVG HashTag', 'SUM HashTag',\n",
      "       'Ratio HashTag', 'SUM Url', 'AVG Url', 'RATIO Url', 'SUM Mention',\n",
      "       'AVG Mention', 'Ratio Mention', 'AVG Statues', 'AVG Listed',\n",
      "       'AVG Follower', 'AVG followers/friend', 'AVG favorite', 'Tweets Count',\n",
      "       'Ratio Verified', 'SUM Verified', 'SUM RT', 'AVG RT', 'AVG AccAge',\n",
      "       'thread_time', 'AVG Emoji', 'RATIO Emoji', 'Ratio Media',\n",
      "       'RATIO Question', 'RATIO Exclaim', 'RATIO Period', 'AVG FPP', 'AVG SPP',\n",
      "       'AVG TPP', 'AVG Skepticism', 'Ratio Skepticism', 'test_auxiliary',\n",
      "       'test_tentat', 'test_certain', 'root_user_ratio', 'unique_user_ratio',\n",
      "       'unique_user_sum', 'NodeThreadCount_0', 'NodeThreadCount_25',\n",
      "       'NodeThreadCount_5', 'NodeThreadCount_75', 'NodeThreadRatio_0',\n",
      "       'NodeThreadRatio_25', 'NodeThreadRatio_5', 'NodeThreadRatio_75'],\n",
      "      dtype='object') \n",
      "\n",
      "pheme_bert_simple_normal (623, 768)\n",
      "Index(['BERTEmbed_0', 'BERTEmbed_1', 'BERTEmbed_2', 'BERTEmbed_3',\n",
      "       'BERTEmbed_4', 'BERTEmbed_5', 'BERTEmbed_6', 'BERTEmbed_7',\n",
      "       'BERTEmbed_8', 'BERTEmbed_9',\n",
      "       ...\n",
      "       'BERTEmbed_758', 'BERTEmbed_759', 'BERTEmbed_760', 'BERTEmbed_761',\n",
      "       'BERTEmbed_762', 'BERTEmbed_763', 'BERTEmbed_764', 'BERTEmbed_765',\n",
      "       'BERTEmbed_766', 'BERTEmbed_767'],\n",
      "      dtype='object', length=768) \n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(\"pheme_sparse_final\", ext_sparse_final.shape)\n",
    "print(ext_sparse_final.columns, \"\\n\")\n",
    "print(\"pheme_pos_final\", ext_pos_final.shape)\n",
    "x = zip(pheme_pos_final.columns.values, ['Interjection', 'Hashtag', 'coordinating conjunction', 'punctuation', 'at-mention', 'proper noun', 'adjective' 'determiner' 'other', 'nominal + verbal',\n",
    "    'common noun', 'pronoun', 'pre/postposition/subordinating conjunction', 'adverb', 'nominal + possessive', 'verb particle', 'URL or email', 'verb incl. copula, auxiliaries',\n",
    " 'existential there, predeterminers', 'proper noun + possessive', 'discourse marker'])\n",
    "print(set(x), \"\\n\")\n",
    "print(\"pheme_thread_final\", ext_thread_final_avg.shape)\n",
    "print(ext_thread_final_avg.columns, \"\\n\")\n",
    "print(\"pheme_bert_simple_normal\", ext_bert_simple_normal.shape)\n",
    "print(ext_bert_simple_normal.columns, \"\\n\")\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Functions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seed(seed_value=42):\n",
    "    \"\"\"Set seed for reproducibility.\n",
    "    \"\"\"\n",
    "    random.seed(seed_value)\n",
    "    np.random.seed(seed_value)\n",
    "    torch.manual_seed(seed_value)\n",
    "    torch.cuda.manual_seed_all(seed_value)\n",
    "\n",
    "torch.manual_seed(42)\n",
    "torch.cuda.manual_seed(42)\n",
    "np.random.seed(42)\n",
    "random.seed(42)\n",
    "torch.backends.cudnn.deterministic=True\n",
    "torch.backends.cudnn.benchmark = False\n",
    "\n",
    "seed = 42\n",
    "\n",
    "def _init_fn(worker_id):\n",
    "    np.random.seed(int(seed))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getDataSize(tensor_x1, tensor_y1, tensor_x2, tensor_y2):\n",
    "    train_size = int(tensor_y1.size(0))\n",
    "    test_size = int(tensor_y2.size(0))\n",
    "\n",
    "    # print(\"Variables)\\n\\tTrain:%s\\n\\tTest: %s\"%(tensor_x1.size(),tensor_x2.size()))\n",
    "    # print(\"\\tTargets:%s \\ %s\"%(tensor_y1.size()[0],tensor_y2.size()[0]))\n",
    "    # print(\"Train Size\",train_size,\"Test Size\",test_size)\n",
    "    # print()\n",
    "    return train_size, test_size\n",
    "\n",
    "def reset_weights(m):\n",
    "  '''\n",
    "    Try resetting model weights to avoid\n",
    "    weight leakage.\n",
    "  '''\n",
    "  for layer in m.children():\n",
    "    if hasattr(layer, 'reset_parameters'):\n",
    "      # print(f'Reset trainable parameters of layer = {layer}')\n",
    "      # print(\"Parameter Resetted\")\n",
    "      layer.reset_parameters()\n",
    "\n",
    "class writeLog():\n",
    "  def write(self, fileName, text):\n",
    "    print(text)\n",
    "    f=open(fileName,'a')\n",
    "    f.write(text)\n",
    "    f.write(\"\\n\")\n",
    "    f.close()\n",
    "  def writeWithoutCR(self, fileName, text):\n",
    "    f=open(fileName,'a')\n",
    "    f.write(text)\n",
    "    f.close()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def cv_process(dataset, criterion, modelClass, target, epochs, events, verbose=True, scaling=False):\n",
    "\n",
    "    # cv_pd_list[?][0]은 Training cv_pd_list[?][1] Testing\n",
    "    cv_pd_list = []\n",
    "    data = pd.concat([dataset, events, target], axis=1)\n",
    "    NUM_EVENT = data.Event.unique().shape[0]\n",
    "    EVENTS = data.Event.unique()\n",
    "    results = {}\n",
    "    # modelClass.__class__\n",
    "\n",
    "    for i, d in enumerate(EVENTS):\n",
    "        df1, df2 = [x for _, x in data.groupby(data['Event'] != d)]\n",
    "        df1.reset_index(inplace=True, drop=True)\n",
    "        df2.reset_index(inplace=True, drop=True)\n",
    "        cv_pd_list.append([df2, df1])\n",
    "    \n",
    "    # for train, test in cv_pd_list:\n",
    "    #     print(\"Train: %s \\ Test: %s\" % (train.shape, test.shape))\n",
    "\n",
    "    log = writeLog()\n",
    "    modelname = modelClass.__name__\n",
    "    PREFIX = \"./Model/\"+modelname+\"_\"\n",
    "    log.write(PREFIX+\"log.txt\",f\"\\nSTARTING TEST of {epochs} EPOCH\\n\")\n",
    "\n",
    "    for index, fold in enumerate(cv_pd_list):\n",
    "\n",
    "        # DATA PREPARATION\n",
    "        train, test = fold\n",
    "        log.writeWithoutCR(PREFIX+\"log.txt\",f\"\\n----------------------------------------------------------------------------\\n> FOLD {int(index)+1}\\n----------------------------------------------------------------------------\")\n",
    "        print(f'> FOLD {int(index)+1}')\n",
    "        train_target = train.pop('target')\n",
    "        train.pop('Event').values\n",
    "        test_target = test.pop('target')\n",
    "        test.pop('Event')\n",
    "\n",
    "        if scaling==True:\n",
    "            scaler = StandardScaler()\n",
    "            train = pd.DataFrame(scaler.fit_transform(train))\n",
    "            test = pd.DataFrame(scaler.transform(test))\n",
    "\n",
    "        tensor_x1, tensor_y1, tensor_x2, tensor_y2 = __MLP.convert_df_to_unsqueezed_tensor(\n",
    "            train.values, train_target, test.values, test_target.values)\n",
    "        train_dataset = TensorDataset(tensor_x1, tensor_y1)\n",
    "        test_dataset = TensorDataset(tensor_x2, tensor_y2)\n",
    "\n",
    "        batch_size = 8\n",
    "\n",
    "        # train_sampler, test_sampler = __MLP.getSamplers(train_target, tensor_x2)\n",
    "        counts = np.bincount(train_target.values)\n",
    "        labels_weights = 1. / counts\n",
    "        weights = labels_weights[train_target.values]\n",
    "        train_sampler = WeightedRandomSampler(weights, len(weights))\n",
    "        test_sampler = SequentialSampler(tensor_x2)\n",
    "\n",
    "\n",
    "        train_dataloader = DataLoader(train_dataset, batch_size=batch_size,\n",
    "                                    sampler=train_sampler, pin_memory=True, num_workers=0, worker_init_fn=_init_fn)\n",
    "        test_dataloader = DataLoader(test_dataset, batch_size=batch_size,\n",
    "                                    shuffle=False, pin_memory=True, num_workers=0, worker_init_fn=_init_fn)\n",
    "\n",
    "        data = next(iter(train_dataloader))\n",
    "\n",
    "        train_size, test_size = getDataSize(tensor_x1, tensor_y1, tensor_x2, tensor_y2)\n",
    "\n",
    "        if verbose==True:\n",
    "            print(\"mean: %s, std: %s\" % (data[0].mean(), data[0].std()))\n",
    "            print(\"Train Size\",train_size,\"Test Size\",test_size)\n",
    "        log.writeWithoutCR(PREFIX+\"log.txt\", f\"\\nmean: {data[0].mean()}, std: {data[0].std()}\")\n",
    "        log.writeWithoutCR(PREFIX+\"log.txt\", f\"\\nTrain Size: {train_size}, Test Size: {test_size}\\n\")\n",
    "        \n",
    "\n",
    "        model = modelClass()\n",
    "        model.apply(reset_weights)\n",
    "        # for layer in model.children():\n",
    "        #     if hasattr(layer, 'reset_parameters'):\n",
    "        #         print(f'Reset trainable parameters of layer = {layer}')\n",
    "        #         layer.reset_parameters()\n",
    "\n",
    "\n",
    "        # model_sparse = sparse_model()\n",
    "        # criterion = nn.BCEWithLogitsLoss()\n",
    "        # optimizer = optim.SGD(model_sparse.parameters(), lr=0.01, momentum=0.9)\n",
    "        # optimizer = optim.Adam(model_sparse.parameters(), lr=5e-5, eps=1e-8, weight_decay=1e-7)\n",
    "        optimizer = AdamW(model.parameters(),\n",
    "                        lr=1e-4,    # Default learning rate\n",
    "                        # lr=9e-5,    # Default learning rate\n",
    "                        eps=1e-8,    # Default epsilon value\n",
    "                        weight_decay=1e-6\n",
    "                        )\n",
    "\n",
    "\n",
    "        total_steps = len(train_dataloader) * epochs\n",
    "        # print(f'length of tloader: {len(train_dataloader)}')\n",
    "        # print(f'total step: {total_steps}')\n",
    "        log.writeWithoutCR(PREFIX+\"log.txt\", f'total step: {total_steps}\\n')\n",
    "        \n",
    "        scheduler = get_linear_schedule_with_warmup(optimizer,\n",
    "                                                    num_warmup_steps=0,  # Default value\n",
    "                                                    num_training_steps=total_steps)\n",
    "\n",
    "        # print(f'Model: {modelname}')\n",
    "        PATH = \"./Model/\"+modelname+\"_\"+str(index+1)+\".pt\"\n",
    "        # print(f'PATH: {PATH}')\n",
    "        log.writeWithoutCR(PREFIX+\"log.txt\", f'PATH: {PATH}')\n",
    "\n",
    "        training_acc = []\n",
    "        training_loss = []\n",
    "        # train_acc, train_loss, val_acc, val_loss_list = __MLP.train_sequential(model=model, num_epochs=epochs, patience=patience, criterion=criterion, optimizer=optimizer, scheduler=scheduler, train_loader=train_dataloader, train_size=train_size, test_loader=test_dataloader, test_size=test_size, PATH=PATH)\n",
    "\n",
    "        # Run the training loop for defined number of epochs\n",
    "        for epoch in range(0, epochs):\n",
    "\n",
    "            # Print epoch\n",
    "            if (verbose!=False):\n",
    "                # pass\n",
    "                print(f'Starting epoch {epoch+1}')\n",
    "            elif (verbose!=True):\n",
    "                if epoch%50 == 49:\n",
    "                    print(f'Starting epoch {epoch+1}')\n",
    "            # Set current loss value\n",
    "            current_loss = 0.0\n",
    "            running_corrects = 0.0\n",
    "            running_loss = 0.0\n",
    "\n",
    "            # Iterate over the DataLoader for training data\n",
    "            for i, data in enumerate(train_dataloader, 0):\n",
    "\n",
    "                # Get inputs\n",
    "                inputs, targets = data\n",
    "\n",
    "                # Zero the gradients\n",
    "                optimizer.zero_grad()\n",
    "\n",
    "                # Perform forward pass\n",
    "                outputs = model(inputs)\n",
    "\n",
    "                outputs = outputs.view(outputs.size(0), -1)\n",
    "\n",
    "                # Compute Prediction Outputs\n",
    "                # preds = outputs.squeeze(1) > 0.0\n",
    "                preds = outputs > 0.0\n",
    "\n",
    "                # Compute loss\n",
    "                loss = criterion(outputs, targets)\n",
    "\n",
    "                running_loss += loss.item() * inputs.size(0)\n",
    "                running_corrects += torch.sum(preds == targets.data).data\n",
    "\n",
    "                # Perform backward pass\n",
    "                loss.backward()\n",
    "\n",
    "                # Perform optimization and Scheduler\n",
    "                optimizer.step()\n",
    "                scheduler.step()\n",
    "\n",
    "                # Print statistics\n",
    "                # current_loss += loss.item() # 원본\n",
    "                # if i % len(train_dataloader) == len(train_dataloader)-1:\n",
    "                #     print('Loss after mini-batch %5d: %.3f' %\n",
    "                #           (i + 1, current_loss / i+1))\n",
    "\n",
    "                current_loss += loss.item() * inputs.size(0)\n",
    "                if verbose == True:\n",
    "                    if i % len(train_dataloader) == len(train_dataloader)-1:\n",
    "                        print(\"Loss/ACC after mini-batch %5d: %.3f / %.4f\" %\n",
    "                            (i + 1, current_loss / train_size, running_corrects/train_size))\n",
    "\n",
    "            # epoch_acc = running_corrects.double() / train_size\n",
    "            epoch_acc = running_corrects / train_size\n",
    "            epoch_loss = running_loss / train_size\n",
    "            training_acc.append(epoch_acc)\n",
    "            training_loss.append(epoch_loss)\n",
    "            # print('Epoch {}/{}\\tTrain) Acc: {:.4f}, Loss: {:.4f}'.format(epoch+1,\n",
    "                                                                        #  epochs, epoch_acc, epoch_loss))\n",
    "            \n",
    "        # Process is complete.\n",
    "        if verbose==True:\n",
    "            print('>> Training process has finished. Saving trained model and starting Testing')\n",
    "\n",
    "        # Print about testing\n",
    "\n",
    "        # Saving the model\n",
    "        torch.save(model.state_dict(), PATH)\n",
    "\n",
    "        # Evaluation for this fold\n",
    "        correct, total = 0, 0\n",
    "        val_corrects=0\n",
    "        f1_batch_epoch = 0\n",
    "        val_label_list = []\n",
    "        val_loss = 0\n",
    "        with torch.no_grad():\n",
    "\n",
    "            # Iterate over the test data and generate predictions\n",
    "            for i, data in enumerate(test_dataloader, 0):\n",
    "\n",
    "                # Get inputs\n",
    "                inputs, targets = data\n",
    "\n",
    "                # Generate outputs\n",
    "                outputs = model(inputs)\n",
    "\n",
    "                # Set total and correct\n",
    "                outputs = outputs.view(outputs.size(0), -1).float()\n",
    "                predicted = (outputs > 0.0).float()\n",
    "                correct += (predicted == targets).float().sum().item()\n",
    "\n",
    "                loss = criterion(outputs, targets)\n",
    "                val_loss += loss.item() * inputs.size(0)\n",
    "                #!\n",
    "                preds = (outputs > 0.0).float()\n",
    "                # running_loss += loss.item() * inputs.size(0)\n",
    "                val_corrects += torch.sum(preds == targets.data).data\n",
    "                # f1_batch = f1_score(targets.cpu(), outputs.sigmoid().cpu() > 0.5, average='macro')\n",
    "                f1_batch = f1_score(targets.cpu(), preds, average='macro')\n",
    "                f1_batch_epoch += f1_batch * inputs.size(0)\n",
    "                # f1_running += (f1_score(targets, preds, average='macro') * inputs.size(0))\n",
    "                total += targets.size(0)\n",
    "            \n",
    "            if verbose==True:\n",
    "                # Print accuracy\n",
    "                print('Accuracy for fold %d: %f %%' % (index, 100.0 * correct / total))\n",
    "                # print('Accuracy-2 for fold %d: %f %%' % (index, 100.0 * val_corrects / total))\n",
    "                # print('F1 Score-2 for fold %d: %f ->  %%' %(index, f1_score(targets, preds, zero_division=False)))\n",
    "                # print('F1 Score-3 for fold %d: %f %%' %(index, f1_score(targets, predicted, zero_division=False)))\n",
    "                print('F1 Score for fold %d: %f %%' %(index, f1_batch_epoch / total))\n",
    "                print('Loss for fold %d: %f %%' %(index, val_loss / total))\n",
    "                # print('F1 Score-5 for fold %d: %f %%' %(index, f1_batch_epoch / test_size))\n",
    "                # print('F1 Score-6 for fold %d: %f %%' %(index, f1_running / test_size))\n",
    "            \n",
    "            results[index] = [100.0 * (correct / total), 100.0 * f1_batch_epoch / total, val_loss / total, train_size, test_size]\n",
    "            log.writeWithoutCR(PREFIX+\"log.txt\", f'\\nAccuracy for fold {index+1}: {100.0 * correct / total:.3f}')\n",
    "            log.writeWithoutCR(PREFIX+\"log.txt\", f'\\nF1 Score for fold {index+1}: {f1_batch_epoch / total:.3f}')\n",
    "            log.writeWithoutCR(PREFIX+\"log.txt\", f'\\nLoss for fold {index+1}: {val_loss / total:.3f}')\n",
    "\n",
    "    # ---------------------------- Print fold results ---------------------------- #\n",
    "    # log.write(PREFIX+\"log.txt\",f\"FOLD {int(index)+1}\\n----------------------------------------------------------------------------\")\n",
    "\n",
    "    # print(f'K-FOLD CROSS VALIDATION RESULTS FOR {NUM_EVENT} FOLDS')\n",
    "    log.write(PREFIX+\"log.txt\", f'\\n\\n----------------------------------------------------------------------------\\n>>>K-FOLD CROSS VALIDATION RESULTS FOR {NUM_EVENT} FOLDS\\n----------------------------------------------------------------------------')\n",
    "\n",
    "    # print('----------------------------------------------------------------------------')\n",
    "    acc_sum = 0.0\n",
    "    f1_sum = 0.0\n",
    "    loss_sum = 0.0\n",
    "    for key, value in results.items():\n",
    "        # print(f'Fold {key}: Acc {value[0]}, F1 {value[1]} %')\n",
    "        log.write(PREFIX+\"log.txt\", f'Fold {key}: Acc {value[0]}, F1 {value[1]}')\n",
    "        acc_sum += value[0]\n",
    "        f1_sum += value[1]\n",
    "        loss_sum += value[2]\n",
    "    # print(f'Average: {acc_sum/len(results.items())} %')\n",
    "    # print(f'F1: {f1_sum/len(results.items())} %')\n",
    "    log.write(PREFIX+\"log.txt\", f'Average: {acc_sum/len(results.items()):.3f}%')\n",
    "    log.write(PREFIX+\"log.txt\", f'F1: {f1_sum/len(results.items()):.3f}%')\n",
    "    log.write(PREFIX+\"log.txt\", f'Loss: {loss_sum/len(results.items()):.3f}\\n')\n",
    "\n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def epochs_diff(testing_results):\n",
    "    string = \"\"\n",
    "    print('----------------------------------------------------------------------------')\n",
    "    for loop in testing_results:\n",
    "        acc_sum = 0.0\n",
    "        f1_sum = 0.0\n",
    "        loss=0.0\n",
    "        for key, value in loop.items():\n",
    "            print(f'Fold {key}: Acc {value[0]:.2f}%, F1 {value[1]:.2f}%, Loss {value[2]:.2f}')\n",
    "            acc_sum += value[0]\n",
    "            f1_sum += value[1]\n",
    "            loss += value[2]\n",
    "        string+=f'Average: {acc_sum/len(loop.items()):.2f}%\\n'\n",
    "        string+=f'F1: {f1_sum/len(loop.items()):.2f}%\\n'\n",
    "        string+=f'Loss: {loss/len(loop.items()):.2f}\\n'\n",
    "        # print(f'Average: {acc_sum/len(loop.items()):.2f} %')\n",
    "        # print(f'F1: {f1_sum/len(loop.items()):.2f} %')\n",
    "        print(\"-----------------------------\")\n",
    "        return string"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch (OLD)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [100] epochs\n",
      "************************************************************\n",
      "TIME: 31/03/2021 21:34:10\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=49, out_features=16, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=16, out_features=5, bias=True)\n",
      "  (4): ELU(alpha=1.0)\n",
      "  (5): Dropout(p=0.5, inplace=False)\n",
      "  (6): Linear(in_features=5, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 100 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-112-59020d0f4360>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     29\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtesting_epochs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 31\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodelClass\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mroot_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_event\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscaling\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     32\u001b[0m     \u001b[0mtesting_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     33\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-106-12b23e8528e6>\u001b[0m in \u001b[0;36mcv_process\u001b[0;34m(dataset, criterion, modelClass, target, epochs, events, verbose, scaling)\u001b[0m\n\u001b[1;32m    125\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    126\u001b[0m             \u001b[0;31m# Iterate over the DataLoader for training data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 127\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    128\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    129\u001b[0m                 \u001b[0;31m# Get inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    518\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 557\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    558\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     42\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     43\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mauto_collation\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 44\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0midx\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0midx\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataset.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, index)\u001b[0m\n\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    172\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataset.py\u001b[0m in \u001b[0;36m<genexpr>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m    169\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    170\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__getitem__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 171\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtuple\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtensor\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mtensor\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtensors\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    172\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    173\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__len__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "class root_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(root_model, self).__init__()  # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(49, 16),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(16, 5),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(5, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "# model = sparse_model()\n",
    "dataset = all_root\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [100]\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in root_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=root_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [50, 100] epochs\n",
      "************************************************************\n",
      "TIME: 31/03/2021 17:25:36\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=49, out_features=12, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=12, out_features=4, bias=True)\n",
      "  (4): ELU(alpha=1.0)\n",
      "  (5): Dropout(p=0.5, inplace=False)\n",
      "  (6): Linear(in_features=4, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 50 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 73.78547378547378, F1 49.294405363446195\n",
      "Fold 1: Acc 61.15485564304461, F1 37.3151144511986\n",
      "Fold 2: Acc 53.09168443496801, F1 33.74488664436882\n",
      "Fold 3: Acc 54.157303370786515, F1 35.227680370111\n",
      "Fold 4: Acc 54.217854217854224, F1 34.159599099712\n",
      "Fold 5: Acc 14.285714285714285, F1 12.5\n",
      "Fold 6: Acc 74.67811158798283, F1 45.99774339747701\n",
      "Fold 7: Acc 45.79831932773109, F1 31.223268754141976\n",
      "Fold 8: Acc 57.2463768115942, F1 42.19624954547816\n",
      "Average: 54.268%\n",
      "F1: 35.740%\n",
      "Loss: 86.885\n",
      "\n",
      "\n",
      "STARTING TEST of 100 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 23.56902356902357, F1 20.97733601242531\n",
      "Fold 1: Acc 61.76727909011374, F1 37.72010715829083\n",
      "Fold 2: Acc 48.8272921108742, F1 38.500431213055464\n",
      "Fold 3: Acc 54.606741573033716, F1 34.48935521289566\n",
      "Fold 4: Acc 51.35135135135135, F1 32.00788066845551\n",
      "Fold 5: Acc 57.14285714285714, F1 36.36363636363636\n",
      "Fold 6: Acc 50.21459227467812, F1 33.75091457028386\n",
      "Fold 7: Acc 49.57983193277311, F1 32.69319692337595\n",
      "Fold 8: Acc 44.927536231884055, F1 31.54263508353336\n",
      "Average: 49.110%\n",
      "F1: 33.116%\n",
      "Loss: 7.862\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 73.79%, F1 49.29%, Loss 359.59\n",
      "Fold 1: Acc 61.15%, F1 37.32%, Loss 4.50\n",
      "Fold 2: Acc 53.09%, F1 33.74%, Loss 4.42\n",
      "Fold 3: Acc 54.16%, F1 35.23%, Loss 65.91\n",
      "Fold 4: Acc 54.22%, F1 34.16%, Loss 338.18\n",
      "Fold 5: Acc 14.29%, F1 12.50%, Loss 0.89\n",
      "Fold 6: Acc 74.68%, F1 46.00%, Loss 5.04\n",
      "Fold 7: Acc 45.80%, F1 31.22%, Loss 2.59\n",
      "Fold 8: Acc 57.25%, F1 42.20%, Loss 0.85\n",
      "-----------------------------\n"
     ]
    }
   ],
   "source": [
    "class root_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(root_model, self).__init__()  # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(49, 12),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(12, 4),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(4, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "# model = sparse_model()\n",
    "dataset = all_root\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [50, 100]\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in root_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=root_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [50, 100] epochs\n",
      "************************************************************\n",
      "TIME: 31/03/2021 17:30:37\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=49, out_features=8, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=8, out_features=3, bias=True)\n",
      "  (4): ELU(alpha=1.0)\n",
      "  (5): Dropout(p=0.5, inplace=False)\n",
      "  (6): Linear(in_features=3, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 50 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 54.97835497835498, F1 35.23458110574113\n",
      "Fold 1: Acc 44.70691163604549, F1 30.3485136693663\n",
      "Fold 2: Acc 49.25373134328358, F1 34.492233367378645\n",
      "Fold 3: Acc 49.21348314606742, F1 32.297152042094005\n",
      "Fold 4: Acc 51.269451269451274, F1 32.62862890971249\n",
      "Fold 5: Acc 64.28571428571429, F1 39.130434782608695\n",
      "Fold 6: Acc 12.446351931330472, F1 10.656407280930596\n",
      "Fold 7: Acc 50.42016806722689, F1 34.963672749139874\n",
      "Fold 8: Acc 51.449275362318836, F1 33.66582368691841\n",
      "Average: 47.558%\n",
      "F1: 31.491%\n",
      "Loss: 121.558\n",
      "\n",
      "\n",
      "STARTING TEST of 100 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 40.83694083694084, F1 28.042188897053016\n",
      "Fold 1: Acc 50.04374453193351, F1 33.05969730110105\n",
      "Fold 2: Acc 47.54797441364605, F1 30.727148246778064\n",
      "Fold 3: Acc 49.10112359550562, F1 32.800408540752926\n",
      "Fold 4: Acc 57.65765765765766, F1 51.590395419962434\n",
      "Fold 5: Acc 57.14285714285714, F1 36.36363636363636\n",
      "Fold 6: Acc 69.09871244635193, F1 43.80193686314937\n",
      "Fold 7: Acc 49.159663865546214, F1 32.813235497446016\n",
      "Fold 8: Acc 48.55072463768116, F1 32.665882762061216\n",
      "Average: 52.127%\n",
      "F1: 35.763%\n",
      "Loss: 0.847\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 54.98%, F1 35.23%, Loss 57.42\n",
      "Fold 1: Acc 44.71%, F1 30.35%, Loss 5.12\n",
      "Fold 2: Acc 49.25%, F1 34.49%, Loss 359.95\n",
      "Fold 3: Acc 49.21%, F1 32.30%, Loss 1.15\n",
      "Fold 4: Acc 51.27%, F1 32.63%, Loss 456.36\n",
      "Fold 5: Acc 64.29%, F1 39.13%, Loss 0.71\n",
      "Fold 6: Acc 12.45%, F1 10.66%, Loss 0.87\n",
      "Fold 7: Acc 50.42%, F1 34.96%, Loss 0.74\n",
      "Fold 8: Acc 51.45%, F1 33.67%, Loss 211.70\n",
      "-----------------------------\n"
     ]
    }
   ],
   "source": [
    "class root_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(root_model, self).__init__()  # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(49, 8),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(8, 3),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(3, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "# model = sparse_model()\n",
    "dataset = all_root\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [50, 100]\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in root_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=root_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [100] epochs\n",
      "************************************************************\n",
      "TIME: 31/03/2021 22:21:44\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=52, out_features=12, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=12, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 100 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 53.775853775853776, F1 34.449838549715246\n",
      "Fold 1: Acc 48.90638670166229, F1 32.82115447360467\n",
      "Fold 2: Acc 50.53304904051173, F1 33.530573241707\n",
      "Fold 3: Acc 50.89887640449439, F1 33.58396452639668\n",
      "Fold 4: Acc 50.696150696150696, F1 33.384228294568565\n",
      "Fold 5: Acc 35.714285714285715, F1 26.315789473684212\n",
      "Fold 6: Acc 66.95278969957081, F1 41.24797778260853\n",
      "Fold 7: Acc 47.47899159663865, F1 31.764306705407577\n",
      "Fold 8: Acc 55.072463768115945, F1 35.74230385824589\n",
      "Average: 51.114%\n",
      "F1: 33.649%\n",
      "Loss: 1.232\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 53.78%, F1 34.45%, Loss 0.84\n",
      "Fold 1: Acc 48.91%, F1 32.82%, Loss 4.36\n",
      "Fold 2: Acc 50.53%, F1 33.53%, Loss 0.71\n",
      "Fold 3: Acc 50.90%, F1 33.58%, Loss 0.71\n",
      "Fold 4: Acc 50.70%, F1 33.38%, Loss 0.73\n",
      "Fold 5: Acc 35.71%, F1 26.32%, Loss 0.75\n",
      "Fold 6: Acc 66.95%, F1 41.25%, Loss 1.38\n",
      "Fold 7: Acc 47.48%, F1 31.76%, Loss 0.82\n",
      "Fold 8: Acc 55.07%, F1 35.74%, Loss 0.78\n",
      "-----------------------------\n"
     ]
    }
   ],
   "source": [
    "class thread_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(thread_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(52, 12),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(12, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "# model = thread_model()\n",
    "dataset = all_thread\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "# epochs = 10\n",
    "\n",
    "testing_epochs = [100]\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in thread_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=thread_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 's' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-116-ded5ba42480f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0ms\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m: name 's' is not defined"
     ]
    }
   ],
   "source": [
    "s"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [100] epochs\n",
      "************************************************************\n",
      "TIME: 31/03/2021 22:21:29\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=101, out_features=24, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=24, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 100 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-119-c351fdf55d7d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     30\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     31\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtesting_epochs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 32\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodelClass\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_sparse_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_event\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscaling\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     33\u001b[0m     \u001b[0mtesting_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     34\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-118-8019942f25d6>\u001b[0m in \u001b[0;36mcv_process\u001b[0;34m(dataset, criterion, modelClass, target, epochs, events, verbose, scaling)\u001b[0m\n\u001b[1;32m    149\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    150\u001b[0m                 \u001b[0;31m# Perform backward pass\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 151\u001b[0;31m                 \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    152\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    153\u001b[0m                 \u001b[0;31m# Perform optimization and Scheduler\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph, inputs)\u001b[0m\n\u001b[1;32m    243\u001b[0m                 \u001b[0mcreate_graph\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m                 inputs=inputs)\n\u001b[0;32m--> 245\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0minputs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    246\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    247\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables, inputs)\u001b[0m\n\u001b[1;32m    143\u001b[0m         \u001b[0mretain_graph\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    144\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 145\u001b[0;31m     Variable._execution_engine.run_backward(\n\u001b[0m\u001b[1;32m    146\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors_\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0minputs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    147\u001b[0m         allow_unreachable=True, accumulate_grad=True)  # allow_unreachable flag\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "class all_sparse_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(all_sparse_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(101, 24),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(24, 1),\n",
    "            # nn.ELU(),\n",
    "            # nn.Dropout(0.5),\n",
    "            # nn.Linear(6, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "# model = thread_model()\n",
    "dataset = all_root_thread\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "# epochs = 10\n",
    "\n",
    "testing_epochs = [100]\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in all_sparse_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=all_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "# writeLog().writeWithoutCR(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [100] epochs\n",
      "************************************************************\n",
      "\n",
      "STARTING TEST of 100 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 78.73977873977874, F1 45.625470023800524\n",
      "Fold 1: Acc 72.44094488188976, F1 43.50742731422256\n",
      "Fold 2: Acc 68.65671641791045, F1 41.51088821098379\n",
      "Fold 3: Acc 74.6067415730337, F1 44.43060995288757\n",
      "Fold 4: Acc 72.48157248157248, F1 42.85610780556853\n",
      "Fold 5: Acc 85.71428571428571, F1 46.153846153846146\n",
      "Fold 6: Acc 57.08154506437768, F1 37.5900551246698\n",
      "Fold 7: Acc 60.08403361344538, F1 35.96407494406936\n",
      "Fold 8: Acc 55.79710144927537, F1 35.321918530488155\n",
      "Average: 69.511%\n",
      "F1: 41.440%\n",
      "Loss: 0.626\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 78.74%, F1 45.63%, Loss 0.49\n",
      "Fold 1: Acc 72.44%, F1 43.51%, Loss 0.58\n",
      "Fold 2: Acc 68.66%, F1 41.51%, Loss 0.63\n",
      "Fold 3: Acc 74.61%, F1 44.43%, Loss 0.54\n",
      "Fold 4: Acc 72.48%, F1 42.86%, Loss 0.60\n",
      "Fold 5: Acc 85.71%, F1 46.15%, Loss 0.35\n",
      "Fold 6: Acc 57.08%, F1 37.59%, Loss 0.73\n",
      "Fold 7: Acc 60.08%, F1 35.96%, Loss 0.80\n",
      "Fold 8: Acc 55.80%, F1 35.32%, Loss 0.90\n",
      "-----------------------------\n"
     ]
    },
    {
     "data": {
      "text/plain": "'Average: 69.51%\\nF1: 41.44%\\nLoss: 0.63\\n'"
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "class bert_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(bert_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(768, 50, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(50, 8, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(8, 1)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "# model = bert_model()\n",
    "dataset = all_bert_simple\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "# epochs = 10\n",
    "\n",
    "testing_epochs = [100]\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "epochs_diff(testing_results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [100, 200] epochs\n",
      "************************************************************\n",
      "TIME: 31/03/2021 21:43:35\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=869, out_features=72, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=72, out_features=10, bias=True)\n",
      "  (4): ELU(alpha=1.0)\n",
      "  (5): Dropout(p=0.5, inplace=False)\n",
      "  (6): Linear(in_features=10, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 100 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 50.55315055315055, F1 33.257280404346574\n",
      "Fold 1: Acc 49.86876640419948, F1 32.94471483295705\n",
      "Fold 2: Acc 48.8272921108742, F1 32.30743150493777\n",
      "Fold 3: Acc 51.01123595505618, F1 33.588206113064516\n",
      "Fold 4: Acc 49.385749385749385, F1 32.88896584482881\n",
      "Fold 5: Acc 50.0, F1 33.33333333333333\n",
      "Fold 6: Acc 52.36051502145923, F1 34.461065214564\n",
      "Fold 7: Acc 52.52100840336135, F1 34.92412003482518\n",
      "Fold 8: Acc 58.69565217391305, F1 37.4393139294167\n",
      "Average: 51.469%\n",
      "F1: 33.905%\n",
      "Loss: 0.950\n",
      "\n",
      "\n",
      "STARTING TEST of 200 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 2\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 3\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 4\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 5\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 6\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 7\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 8\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "> FOLD 9\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Parameter Resetted\n",
      "Starting epoch 50\n",
      "Starting epoch 100\n",
      "Starting epoch 150\n",
      "Starting epoch 200\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 49.254449254449256, F1 32.71179264550313\n",
      "Fold 1: Acc 49.95625546806649, F1 32.886820729136524\n",
      "Fold 2: Acc 48.40085287846482, F1 32.482880315603616\n",
      "Fold 3: Acc 51.79775280898876, F1 34.14379168751419\n",
      "Fold 4: Acc 48.730548730548726, F1 32.41808221022559\n",
      "Fold 5: Acc 42.857142857142854, F1 30.0\n",
      "Fold 6: Acc 54.506437768240346, F1 35.60881207904906\n",
      "Fold 7: Acc 52.94117647058824, F1 35.35089557595952\n",
      "Fold 8: Acc 50.72463768115942, F1 34.29147039377218\n",
      "Average: 49.908%\n",
      "F1: 33.322%\n",
      "Loss: 0.709\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 50.55%, F1 33.26%, Loss 2.13\n",
      "Fold 1: Acc 49.87%, F1 32.94%, Loss 0.73\n",
      "Fold 2: Acc 48.83%, F1 32.31%, Loss 0.71\n",
      "Fold 3: Acc 51.01%, F1 33.59%, Loss 0.74\n",
      "Fold 4: Acc 49.39%, F1 32.89%, Loss 1.41\n",
      "Fold 5: Acc 50.00%, F1 33.33%, Loss 0.71\n",
      "Fold 6: Acc 52.36%, F1 34.46%, Loss 0.70\n",
      "Fold 7: Acc 52.52%, F1 34.92%, Loss 0.70\n",
      "Fold 8: Acc 58.70%, F1 37.44%, Loss 0.72\n",
      "-----------------------------\n"
     ]
    }
   ],
   "source": [
    "class bert_sparse_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(bert_sparse_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(869, 72, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(72, 10, bias = True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(10, 1)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "# model = bert_model()\n",
    "dataset = all_total\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "# epochs = 10\n",
    "\n",
    "testing_epochs = [100,200]\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in bert_sparse_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "epochs_diff(testing_results)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[{0: [30.014430014430015, 20.329881502230815, 0.9727699329441358],\n  1: [52.31846019247593, 34.094144605369294, 0.7706452375530467],\n  2: [52.23880597014925, 33.81135426518758, 1.5158748644501416],\n  3: [51.348314606741575, 33.655633914142015, 0.7604606889606862],\n  4: [47.338247338247335, 31.74001647471132, 5.580640545299461],\n  5: [50.0, 33.33333333333333, 0.6937494874000549],\n  6: [42.48927038626609, 29.919186399219985, 0.737345610821196],\n  7: [50.84033613445378, 33.669406786111594, 0.7304117864921313],\n  8: [52.17391304347826, 35.17515949242477, 0.7004137376080388]}]"
     },
     "execution_count": 98,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "testing_results"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Batch"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import AutoModel, AutoTokenizer\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "class InputExample(object):\n",
    "    def __init__(self, id, text, labels=None):\n",
    "        self.id = id\n",
    "        self.text = text\n",
    "        self.labels = labels\n",
    "\n",
    "class InputFeatures(object):\n",
    "    def __init__(self, input_ids, input_mask, segment_ids, label_ids):\n",
    "        self.input_ids = input_ids\n",
    "        self.input_mask = input_mask\n",
    "        self.segment_ids = segment_ids\n",
    "        self.label_ids = label_ids\n",
    "\n",
    "def get_train_examples(train_file):\n",
    "    train_df = pd.read_csv(train_file)\n",
    "    ids = train_df['id'].values\n",
    "    text = train_df['comment_text'].values\n",
    "    labels = train_df[train_df.columns[2:]].values\n",
    "    examples = []\n",
    "    for i in range(len(train_df)):\n",
    "        examples.append(InputExample(ids[i], text[i], labels=labels[i]))\n",
    "    return examples\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_features_from_examples(examples, max_seq_len, tokenizer):\n",
    "    features = []\n",
    "    for i, example in enumerate(examples):\n",
    "        tokens = tokenizer.tokenize(examples.text[i])\n",
    "        if len(tokens) > max_seq_len - 2:\n",
    "            tokens = tokens[:(max_seq_len - 2)]\n",
    "        tokens = [\"[CLS]\"] + tokens + [\"[SEP]\"]\n",
    "        input_ids = tokenizer.convert_tokens_to_ids(tokens)\n",
    "        input_mask = [1] * len(input_ids)\n",
    "        segment_ids = [0] * len(tokens)\n",
    "        padding = [0] * (max_seq_len - len(input_ids))\n",
    "        input_ids += padding\n",
    "        input_mask += padding\n",
    "        segment_ids += padding\n",
    "        assert len(input_ids) == max_seq_len\n",
    "        assert len(input_mask) == max_seq_len\n",
    "        assert len(segment_ids) == max_seq_len\n",
    "        label_ids = [float(target) for target in examples.target]\n",
    "        features.append(InputFeatures(input_ids=input_ids,\n",
    "                                      input_mask=input_mask,\n",
    "                                      segment_ids=segment_ids,\n",
    "                                      label_ids=label_ids))\n",
    "    return features\n",
    "\n",
    "def get_dataset_from_features(features):\n",
    "    input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)\n",
    "    input_mask = torch.tensor([f.input_mask for f in features], dtype=torch.long)\n",
    "    segment_ids = torch.tensor([f.segment_ids for f in features], dtype=torch.long)\n",
    "    label_ids = torch.tensor([f.label_ids for f in features], dtype=torch.float)\n",
    "    dataset = TensorDataset(input_ids,\n",
    "                            input_mask,\n",
    "                            segment_ids,\n",
    "                            label_ids)\n",
    "    return dataset\n",
    "\n",
    "def text_preprocessing(text): # Create a function to tokenize a set of texts\n",
    "\n",
    "    \"\"\"\n",
    "    - Remove entity mentions (eg. '@united')\n",
    "    - Correct errors (eg. '&amp;' to '&')\n",
    "    @param    text (str): a string to be processed.\n",
    "    @return   text (Str): the processed string.\n",
    "    \"\"\"\n",
    "\n",
    "    # text = re.sub(r\"http\\S+\", \"*\", text)  # http link -> '*'\n",
    "    # sent = re.sub(r'([^\\s\\w@#\\*]|_)+', '', sent) # Erasing Special Characters\n",
    "\n",
    "    text = emoji.demojize(text)\n",
    "    text = re.sub(r':[^:\\s]*:', r' \\g<0>', text)  # http link -> '*'\n",
    "\n",
    "    # text = re.sub(r':[^:\\s]*(?:::[^:\\s]*)*:', r' \\g<0> ', text)  # http link -> '*'\n",
    "\n",
    "    # text = re.sub(r\"\\n\", \" \", text)   # mention -> '@'\n",
    "    text = re.sub(r\"http\\S+\", \"HTTPURL\", text)  # http link -> '*'\n",
    "    # text = re.sub(r\"@\\S+\", \"@USER\", text)   # mention -> '@'\n",
    "\n",
    "    text = re.sub(r\"@[A-Za-z0-9]+\", \"@USER\", text)   # mention -> '@'\n",
    "\n",
    "    # Remove trailing whitespace\n",
    "    text = re.sub(r'\\s+', ' ', text).strip()\n",
    "    text = re.sub(r'&amp;', '&', text)\n",
    "    # text = tweetTokenizer.tokenize(text)\n",
    "    # text = [emoji.demojize(token) for token in text]\n",
    "\n",
    "    return text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "class KimCNN(nn.Module):\n",
    "    def __init__(self, embed_num, embed_dim, dropout=0.1, kernel_num=3, kernel_sizes=[2,3,4], num_labels=2):\n",
    "        super().__init__()\n",
    "        self.num_labels = num_labels\n",
    "        self.embed_num = embed_num\n",
    "        self.embed_dim = embed_dim\n",
    "        self.dropout = dropout\n",
    "        self.kernel_num = kernel_num\n",
    "        self.kernel_sizes = kernel_sizes\n",
    "        self.embed = nn.Embedding(self.embed_num, self.embed_dim)\n",
    "        self.convs = nn.ModuleList([nn.Conv2d(1, self.kernel_num, (k, self.embed_dim)) for k in self.kernel_sizes])\n",
    "        self.dropout = nn.Dropout(self.dropout)\n",
    "        self.classifier = nn.Linear(len(self.kernel_sizes)*self.kernel_num, self.num_labels)\n",
    "        \n",
    "    def forward(self, inputs, labels=None):\n",
    "        output = inputs.unsqueeze(1)\n",
    "        output = [nn.functional.relu(conv(output)).squeeze(3) for conv in self.convs]\n",
    "        output = [nn.functional.max_pool1d(i, i.size(2)).squeeze(2) for i in output]\n",
    "        output = torch.cat(output, 1)\n",
    "        output = self.dropout(output)\n",
    "        logits = self.classifier(output)\n",
    "        if labels is not None:\n",
    "            loss_fct = nn.BCEWithLogitsLoss()\n",
    "            loss = loss_fct(logits.view(-1, self.num_labels), labels.view(-1, self.num_labels))\n",
    "            return loss\n",
    "        else:\n",
    "            return logits\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "bertweet = AutoModel.from_pretrained(\"vinai/bertweet-base\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False, normalization=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "device = torch.device(type='cuda')\n",
    "# pretrained_weights = \"vinai/bertweet-base\"\n",
    "basemodel = AutoModel.from_pretrained(\"vinai/bertweet-base\")\n",
    "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False, normalization=True)\n",
    "\n",
    "# tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
    "# basemodel = AutoModel.from_pretrained(pretrained_weights)\n",
    "# basemodel.to(device)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_input = pd.concat([pheme_textonly, pheme_y],axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>text</th>\n      <th>target</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>BREAKING: Armed man takes hostage in kosher gr...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>#CharlieHebdo killers dead, confirmed by genda...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>Top French cartoonists Charb, Cabu, Wolinski, ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>Police have surrounded the area where the #Cha...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>PHOTO: Armed gunmen face police officers near ...</td>\n      <td>1</td>\n    </tr>\n    <tr>\n      <th>...</th>\n      <td>...</td>\n      <td>...</td>\n    </tr>\n    <tr>\n      <th>5797</th>\n      <td>'I'll ride with you' http://t.co/llZnuCAzg5 Au...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5798</th>\n      <td>Canada's thoughts and prayers are with our Aus...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5799</th>\n      <td>Every non-muslim in the world must watch this ...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5800</th>\n      <td>Suspect in Sydney cafe siege identified as Man...</td>\n      <td>0</td>\n    </tr>\n    <tr>\n      <th>5801</th>\n      <td>Australians respond to racism by telling #Musl...</td>\n      <td>0</td>\n    </tr>\n  </tbody>\n</table>\n<p>5802 rows × 2 columns</p>\n</div>",
      "text/plain": "                                                   text  target\n0     BREAKING: Armed man takes hostage in kosher gr...       1\n1     #CharlieHebdo killers dead, confirmed by genda...       1\n2     Top French cartoonists Charb, Cabu, Wolinski, ...       1\n3     Police have surrounded the area where the #Cha...       1\n4     PHOTO: Armed gunmen face police officers near ...       1\n...                                                 ...     ...\n5797  'I'll ride with you' http://t.co/llZnuCAzg5 Au...       0\n5798  Canada's thoughts and prayers are with our Aus...       0\n5799  Every non-muslim in the world must watch this ...       0\n5800  Suspect in Sydney cafe siege identified as Man...       0\n5801  Australians respond to racism by telling #Musl...       0\n\n[5802 rows x 2 columns]"
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'str' object has no attribute 'text'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-95-eefbb0d78f79>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;34m,\u001b[0m\u001b[0mexample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_input\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;31m# print(train_input.iloc[i])\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m     \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexample\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m     \u001b[0;31m# if len(tokens) > max_seq_len - 2:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;31m#     tokens = tokens[:(max_seq_len - 2)]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'str' object has no attribute 'text'"
     ]
    }
   ],
   "source": [
    "features = []\n",
    "for i ,example in enumerate(train_input):\n",
    "    # print(train_input.iloc[i])\n",
    "    tokens = tokenizer.tokenize(example.text)\n",
    "    # if len(tokens) > max_seq_len - 2:\n",
    "    #     tokens = tokens[:(max_seq_len - 2)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "ename": "AttributeError",
     "evalue": "'str' object has no attribute 'text'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mAttributeError\u001b[0m                            Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-99-c5a2342a0f5e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtrain_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_features_from_examples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_input\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseq_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m<ipython-input-98-ab3a5c5510db>\u001b[0m in \u001b[0;36mget_features_from_examples\u001b[0;34m(examples, max_seq_len, tokenizer)\u001b[0m\n\u001b[1;32m      2\u001b[0m     \u001b[0mfeatures\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m     \u001b[0;32mfor\u001b[0m \u001b[0mexample\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mexamples\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m         \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtokenize\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mexample\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtext\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mlen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtokens\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0mmax_seq_len\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m             \u001b[0mtokens\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtokens\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmax_seq_len\u001b[0m \u001b[0;34m-\u001b[0m \u001b[0;36m2\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mAttributeError\u001b[0m: 'str' object has no attribute 'text'"
     ]
    }
   ],
   "source": [
    "train_features = get_features_from_examples(train_input, seq_len, tokenizer)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "[<__main__.InputFeatures at 0x7f912318c910>,\n <__main__.InputFeatures at 0x7f9123214f10>]"
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyError",
     "evalue": "'id'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   3079\u001b[0m             \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3080\u001b[0;31m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3081\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/index.pyx\u001b[0m in \u001b[0;36mpandas._libs.index.IndexEngine.get_loc\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/hashtable_class_helper.pxi\u001b[0m in \u001b[0;36mpandas._libs.hashtable.PyObjectHashTable.get_item\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'id'",
      "\nThe above exception was the direct cause of the following exception:\n",
      "\u001b[0;31mKeyError\u001b[0m                                  Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-35-5ad4cbb03e3e>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0mseq_len\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m256\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0mtrain_file\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m'./data/_PHEME_textonly.csv'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0mtrain_examples\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_train_examples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m \u001b[0mtrain_features\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_features_from_examples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_examples\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mseq_len\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtokenizer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0mtrain_dataset\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mget_dataset_from_features\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_features\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-14-cfeb2df34c5e>\u001b[0m in \u001b[0;36mget_train_examples\u001b[0;34m(train_file)\u001b[0m\n\u001b[1;32m     14\u001b[0m \u001b[0;32mdef\u001b[0m \u001b[0mget_train_examples\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     15\u001b[0m     \u001b[0mtrain_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_file\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 16\u001b[0;31m     \u001b[0mids\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     17\u001b[0m     \u001b[0mtext\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'comment_text'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     18\u001b[0m     \u001b[0mlabels\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mtrain_df\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mtrain_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m2\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/core/frame.py\u001b[0m in \u001b[0;36m__getitem__\u001b[0;34m(self, key)\u001b[0m\n\u001b[1;32m   3022\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnlevels\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3023\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_getitem_multilevel\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3024\u001b[0;31m             \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcolumns\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3025\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mis_integer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3026\u001b[0m                 \u001b[0mindexer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mindexer\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/core/indexes/base.py\u001b[0m in \u001b[0;36mget_loc\u001b[0;34m(self, key, method, tolerance)\u001b[0m\n\u001b[1;32m   3080\u001b[0m                 \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget_loc\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcasted_key\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3081\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mKeyError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 3082\u001b[0;31m                 \u001b[0;32mraise\u001b[0m \u001b[0mKeyError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkey\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfrom\u001b[0m \u001b[0merr\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   3083\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3084\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mtolerance\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyError\u001b[0m: 'id'"
     ]
    }
   ],
   "source": [
    "seq_len = 256\n",
    "train_file = './data/_PHEME_textonly.csv'\n",
    "train_examples = get_train_examples(train_file)\n",
    "train_features = get_features_from_examples(train_examples, seq_len, tokenizer)\n",
    "train_dataset = get_dataset_from_features(train_features)\n",
    "\n",
    "\n",
    "train_val_split = 0.1\n",
    "train_size = int(len(train_dataset)*(1-train_val_split))\n",
    "val_size = len(train_dataset) - train_size\n",
    "train_dataset, val_dataset = torch.utils.data.random_split(train_dataset, [train_size, val_size])\n",
    "\n",
    "batch = 8\n",
    "train_sampler = RandomSampler(train_dataset)\n",
    "train_dataloader = DataLoader(train_dataset, sampler=train_sampler, batch_size=batch)\n",
    "val_sampler = SequentialSampler(val_dataset)\n",
    "val_dataloader = DataLoader(val_dataset, sampler=val_sampler, batch_size=batch)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from transformers import (\n",
    "    AutoConfig,\n",
    "    AutoModelForSequenceClassification,\n",
    "    AutoTokenizer,\n",
    "    EvalPrediction,\n",
    "    HfArgumentParser,\n",
    "    Trainer,\n",
    "    TrainingArguments,\n",
    "    set_seed,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'Trainer' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-5d805e7c4000>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m trainer = Trainer(\n\u001b[0m\u001b[1;32m      2\u001b[0m         \u001b[0mmodel\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mmodel\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtraining_args\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m         \u001b[0mtrain_dataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mtrain_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0meval_dataset\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0meval_dataset\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'Trainer' is not defined"
     ]
    }
   ],
   "source": [
    "trainer = Trainer(\n",
    "        model=model,\n",
    "        args=training_args,\n",
    "        train_dataset=train_dataset,\n",
    "        eval_dataset=eval_dataset,\n",
    "        compute_metrics=compute_metrics,\n",
    "    )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Class"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 101,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime\n",
    "torch.manual_seed(42)\n",
    "torch.cuda.manual_seed(42)\n",
    "np.random.seed(42)\n",
    "random.seed(42)\n",
    "torch.backends.cudnn.deterministic=True\n",
    "torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'nn' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-0912142a490d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# 49-16-5-1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m# 49-12-4-1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m \u001b[0;32mclass\u001b[0m \u001b[0mroot_model\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnn\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mModule\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      4\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m         \u001b[0msuper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mroot_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__init__\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# 1*20\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'nn' is not defined"
     ]
    }
   ],
   "source": [
    "# 49-16-5-1\n",
    "# 49-12-4-1\n",
    "class root_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(root_model, self).__init__()  # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(49, 8),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(8, 3),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(3, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "class w2v_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(w2v_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(200, 36, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(36, 8, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(8, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "class bert_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(bert_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(768, 50, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(50, 8, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(8, 1)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "        \n",
    "#101-36-1\n",
    "class all_sparse_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(all_sparse_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(101, 24, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(24, 6, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(6, 1)\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "class bert_w2v_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(bert_w2v_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(968, 64, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            # nn.Linear(100, 12, bias=True),\n",
    "            # nn.ELU(),\n",
    "            # nn.Dropout(0.5),\n",
    "            nn.Linear(64, 12),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(12, 1)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "\n",
    "class bert_sparse_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(bert_sparse_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(869, 72, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(72, 10, bias=True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(10, 1)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "class bert_sparse_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(bert_sparse_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(869, 80, bias = True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(80, 16, bias = True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(16, 1)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x\n",
    "\n",
    "class w2v_sparse_model(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(w2v_sparse_model, self).__init__() # 1*20\n",
    "        self.layers = nn.Sequential(\n",
    "            nn.Linear(301, 31, bias = True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(31, 4, bias = True),\n",
    "            nn.ELU(),\n",
    "            nn.Dropout(0.5),\n",
    "            nn.Linear(4, 1)\n",
    "        )\n",
    "    def forward(self, x):\n",
    "        x = self.layers(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batch 5\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "metadata": {},
   "outputs": [],
   "source": [
    "result_list_5 = []\n",
    "result_mean_list_5 = []\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "testing_epochs = [100]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROOT + POS  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                  ROOT + POS                                  #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_root\n",
    "\n",
    "testing_epochs = [2]\n",
    "testing_epochs = testing_epochs\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "# writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in root_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=root_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RP'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in root_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### THREAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                    THREAD                                    #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_thread\n",
    "\n",
    "testing_epochs = [5]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in thread_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=thread_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'Thread'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in thread_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                     BERT                                     #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_bert_simple\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [5]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+bert_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+bert_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in bert_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+all_bert_simple.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'BERT'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in bert_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### W2V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                     W2V                                      #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_AVGw2v\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [50]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in w2v_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=w2v_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'W2V'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in w2v_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT + W2V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 64,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [50] epochs\n",
      "************************************************************\n",
      "TIME: 01/04/2021 12:38:57\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=968, out_features=64, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=64, out_features=12, bias=True)\n",
      "  (4): ELU(alpha=1.0)\n",
      "  (5): Dropout(p=0.5, inplace=False)\n",
      "  (6): Linear(in_features=12, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 50 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Starting epoch 50\n",
      "> FOLD 2\n",
      "Starting epoch 50\n",
      "> FOLD 3\n",
      "Starting epoch 50\n",
      "> FOLD 4\n",
      "Starting epoch 50\n",
      "> FOLD 5\n",
      "Starting epoch 50\n",
      "> FOLD 6\n",
      "Starting epoch 50\n",
      "> FOLD 7\n",
      "Starting epoch 50\n",
      "> FOLD 8\n",
      "Starting epoch 50\n",
      "> FOLD 9\n",
      "Starting epoch 50\n",
      "\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      ">>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 76.57527657527658, F1 50.07521578950159\n",
      "Fold 1: Acc 71.30358705161855, F1 46.6596318039888\n",
      "Fold 2: Acc 67.16417910447761, F1 41.538262732292544\n",
      "Fold 3: Acc 70.4494382022472, F1 44.65465620521806\n",
      "Fold 4: Acc 71.99017199017199, F1 44.872401236037675\n",
      "Fold 5: Acc 78.57142857142857, F1 64.83516483516483\n",
      "Fold 6: Acc 34.763948497854074, F1 25.44210200862561\n",
      "Fold 7: Acc 61.76470588235294, F1 50.55096164339862\n",
      "Fold 8: Acc 58.69565217391305, F1 38.519258519258514\n",
      "Average: 65.698%\n",
      "F1: 45.239%\n",
      "Loss: 0.727\n",
      "\n",
      "----------------------------------------------------------------------------\n",
      "Fold 0: Acc 76.58%, F1 50.08%, Loss 0.52\n",
      "Fold 1: Acc 71.30%, F1 46.66%, Loss 0.62\n",
      "Fold 2: Acc 67.16%, F1 41.54%, Loss 0.66\n",
      "Fold 3: Acc 70.45%, F1 44.65%, Loss 0.62\n",
      "Fold 4: Acc 71.99%, F1 44.87%, Loss 0.64\n",
      "Fold 5: Acc 78.57%, F1 64.84%, Loss 0.44\n",
      "Fold 6: Acc 34.76%, F1 25.44%, Loss 1.29\n",
      "Fold 7: Acc 61.76%, F1 50.55%, Loss 0.86\n",
      "Fold 8: Acc 58.70%, F1 38.52%, Loss 0.89\n",
      "-----------------------------\n"
     ]
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                  BERT + W2V                                  #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_w2v_bert\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [50]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in bert_w2v_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_w2v_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'BERT + W2V'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in bert_w2v_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROOT + POS + THREAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "\n\n************************************************************\nSTARTING A CROSS-VALIDATION ... [100] epochs\n************************************************************\nTIME: 01/04/2021 10:16:23\nLAYER: [Sequential(\n  (0): Linear(in_features=101, out_features=24, bias=True)\n  (1): ELU(alpha=1.0)\n  (2): Dropout(p=0.5, inplace=False)\n  (3): Linear(in_features=24, out_features=6, bias=True)\n  (4): ELU(alpha=1.0)\n  (5): Dropout(p=0.5, inplace=False)\n  (6): Linear(in_features=6, out_features=1, bias=True)\n)]\n\nSTARTING TEST of 100 EPOCH\n\n> FOLD 1\nStarting epoch 50\nStarting epoch 100\n> FOLD 2\nStarting epoch 50\nStarting epoch 100\n> FOLD 3\nStarting epoch 50\nStarting epoch 100\n> FOLD 4\nStarting epoch 50\nStarting epoch 100\n> FOLD 5\nStarting epoch 50\nStarting epoch 100\n> FOLD 6\nStarting epoch 50\nStarting epoch 100\n> FOLD 7\nStarting epoch 50\nStarting epoch 100\n> FOLD 8\nStarting epoch 50\nStarting epoch 100\n> FOLD 9\nStarting epoch 50\nStarting epoch 100\n\n\n----------------------------------------------------------------------------\n>>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n----------------------------------------------------------------------------\nFold 0: Acc 49.735449735449734, F1 32.3814270278917\nFold 1: Acc 51.70603674540683, F1 33.32614114766349\nFold 2: Acc 46.26865671641791, F1 30.746405927642595\nFold 3: Acc 49.10112359550562, F1 32.24378617637044\nFold 4: Acc 50.696150696150696, F1 33.222250267704865\nFold 5: Acc 57.14285714285714, F1 35.06493506493506\nFold 6: Acc 42.06008583690987, F1 29.116262850168425\nFold 7: Acc 48.319327731092436, F1 31.934074329032306\nFold 8: Acc 53.62318840579711, F1 35.24176306785002\nAverage: 49.850%\nF1: 32.586%\nLoss: 0.726\n\n----------------------------------------------------------------------------\nFold 0: Acc 49.74%, F1 32.38%, Loss 0.70\nFold 1: Acc 51.71%, F1 33.33%, Loss 0.77\nFold 2: Acc 46.27%, F1 30.75%, Loss 0.72\nFold 3: Acc 49.10%, F1 32.24%, Loss 0.70\nFold 4: Acc 50.70%, F1 33.22%, Loss 0.72\nFold 5: Acc 57.14%, F1 35.06%, Loss 0.69\nFold 6: Acc 42.06%, F1 29.12%, Loss 0.72\nFold 7: Acc 48.32%, F1 31.93%, Loss 0.83\nFold 8: Acc 53.62%, F1 35.24%, Loss 0.69\n-----------------------------\n"
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                              ROOT + POS + THREAD                             #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_root_thread\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [100]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in all_sparse_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=all_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RPT'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in all_sparse_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RPT + W2V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [50] epochs\n",
      "************************************************************\n",
      "TIME: 01/04/2021 13:34:25\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=301, out_features=31, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=31, out_features=4, bias=True)\n",
      "  (4): ELU(alpha=1.0)\n",
      "  (5): Dropout(p=0.5, inplace=False)\n",
      "  (6): Linear(in_features=4, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 50 EPOCH\n",
      "\n",
      "> FOLD 1\n",
      "Starting epoch 50\n",
      "> FOLD 2\n",
      "Starting epoch 50\n",
      "> FOLD 3\n"
     ]
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                 ALL Features                                 #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_total_w2v\n",
    "\n",
    "testing_epochs = [50]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+w2v_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+w2v_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+w2v_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in w2v_sparse_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=w2v_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "epochs_diff(testing_results)\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RPT + W2V'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in w2v_sparse_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RPT + BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 108,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "************************************************************\n",
      "STARTING A CROSS-VALIDATION ... [5] epochs\n",
      "************************************************************\n",
      "TIME: 01/04/2021 13:24:27\n",
      "LAYER: [Sequential(\n",
      "  (0): Linear(in_features=869, out_features=80, bias=True)\n",
      "  (1): ELU(alpha=1.0)\n",
      "  (2): Dropout(p=0.5, inplace=False)\n",
      "  (3): Linear(in_features=80, out_features=16, bias=True)\n",
      "  (4): ELU(alpha=1.0)\n",
      "  (5): Dropout(p=0.5, inplace=False)\n",
      "  (6): Linear(in_features=16, out_features=1, bias=True)\n",
      ")]\n",
      "\n",
      "STARTING TEST of 5 EPOCH\n",
      "\n",
      "> FOLD 1\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-108-f74f3e1a5457>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtesting_epochs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodelClass\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbert_sparse_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_event\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0mtesting_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m<ipython-input-95-f51df371c9d6>\u001b[0m in \u001b[0;36mcv_process\u001b[0;34m(dataset, criterion, modelClass, target, epochs, events, verbose, scaling)\u001b[0m\n\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0;31m# Iterate over the DataLoader for training data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 126\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m                 \u001b[0;31m# Get inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    518\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 557\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    558\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[0;34m(batch)\u001b[0m\n\u001b[1;32m     81\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     81\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[0;34m(batch)\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mstorage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_new_shared\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__module__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'numpy'\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'str_'\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'string_'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                 ALL Features                                 #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_total_bert\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [5]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in bert_sparse_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "epochs_diff(testing_results)\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RPT + BERT'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in bert_sparse_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = layers\n",
    "\n",
    "result_list_5.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list_5.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Batch 9\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# result_list = []\n",
    "# result_mean_list = []\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "testing_epochs = [50]\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROOT + POS  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                  ROOT + POS                                  #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_root\n",
    "\n",
    "testing_epochs = [2]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "# writeLog().write(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in root_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=root_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+root_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RP'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in root_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### THREAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                    THREAD                                    #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_thread\n",
    "\n",
    "testing_epochs = [5]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in thread_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=thread_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+thread_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'Thread'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in thread_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                     BERT                                     #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_bert_simple\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [5]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+bert_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+bert_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in bert_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+all_bert_simple.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'BERT'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in bert_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### W2V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "\n\n************************************************************\nSTARTING A CROSS-VALIDATION ... [50] epochs\n************************************************************\nTIME: 01/04/2021 12:08:01\nLAYER: [Sequential(\n  (0): Linear(in_features=200, out_features=36, bias=True)\n  (1): ELU(alpha=1.0)\n  (2): Dropout(p=0.5, inplace=False)\n  (3): Linear(in_features=36, out_features=8, bias=True)\n  (4): ELU(alpha=1.0)\n  (5): Dropout(p=0.5, inplace=False)\n  (6): Linear(in_features=8, out_features=1, bias=True)\n)]\n\nSTARTING TEST of 50 EPOCH\n\n> FOLD 1\nStarting epoch 50\n> FOLD 2\nStarting epoch 50\n> FOLD 3\nStarting epoch 50\n> FOLD 4\nStarting epoch 50\n> FOLD 5\nStarting epoch 50\n> FOLD 6\nStarting epoch 50\n> FOLD 7\nStarting epoch 50\n> FOLD 8\nStarting epoch 50\n> FOLD 9\nStarting epoch 50\n\n\n----------------------------------------------------------------------------\n>>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n----------------------------------------------------------------------------\nFold 0: Acc 70.8994708994709, F1 44.90384715492945\nFold 1: Acc 63.77952755905512, F1 39.43861815252898\nFold 2: Acc 60.76759061833689, F1 39.75443651349833\nFold 3: Acc 69.10112359550563, F1 44.97417688428927\nFold 4: Acc 72.48157248157248, F1 47.75074593256415\nFold 5: Acc 85.71428571428571, F1 67.34693877551021\nFold 6: Acc 43.776824034334766, F1 29.831494209176608\nFold 7: Acc 61.76470588235294, F1 41.28627674846163\nFold 8: Acc 63.76811594202898, F1 47.31896123200471\nAverage: 65.784%\nF1: 44.734%\nLoss: 0.665\n\n----------------------------------------------------------------------------\nFold 0: Acc 70.90%, F1 44.90%, Loss 0.64\nFold 1: Acc 63.78%, F1 39.44%, Loss 0.66\nFold 2: Acc 60.77%, F1 39.75%, Loss 0.81\nFold 3: Acc 69.10%, F1 44.97%, Loss 0.64\nFold 4: Acc 72.48%, F1 47.75%, Loss 0.58\nFold 5: Acc 85.71%, F1 67.35%, Loss 0.34\nFold 6: Acc 43.78%, F1 29.83%, Loss 0.88\nFold 7: Acc 61.76%, F1 41.29%, Loss 0.74\nFold 8: Acc 63.77%, F1 47.32%, Loss 0.69\n-----------------------------\n"
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                     W2V                                      #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_AVGw2v\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [50]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in w2v_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=w2v_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+w2v_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'W2V'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in w2v_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### BERT + W2V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "\n\n************************************************************\nSTARTING A CROSS-VALIDATION ... [50] epochs\n************************************************************\nTIME: 01/04/2021 12:38:57\nLAYER: [Sequential(\n  (0): Linear(in_features=968, out_features=64, bias=True)\n  (1): ELU(alpha=1.0)\n  (2): Dropout(p=0.5, inplace=False)\n  (3): Linear(in_features=64, out_features=12, bias=True)\n  (4): ELU(alpha=1.0)\n  (5): Dropout(p=0.5, inplace=False)\n  (6): Linear(in_features=12, out_features=1, bias=True)\n)]\n\nSTARTING TEST of 50 EPOCH\n\n> FOLD 1\nStarting epoch 50\n> FOLD 2\nStarting epoch 50\n> FOLD 3\nStarting epoch 50\n> FOLD 4\nStarting epoch 50\n> FOLD 5\nStarting epoch 50\n> FOLD 6\nStarting epoch 50\n> FOLD 7\nStarting epoch 50\n> FOLD 8\nStarting epoch 50\n> FOLD 9\nStarting epoch 50\n\n\n----------------------------------------------------------------------------\n>>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n----------------------------------------------------------------------------\nFold 0: Acc 76.57527657527658, F1 50.07521578950159\nFold 1: Acc 71.30358705161855, F1 46.6596318039888\nFold 2: Acc 67.16417910447761, F1 41.538262732292544\nFold 3: Acc 70.4494382022472, F1 44.65465620521806\nFold 4: Acc 71.99017199017199, F1 44.872401236037675\nFold 5: Acc 78.57142857142857, F1 64.83516483516483\nFold 6: Acc 34.763948497854074, F1 25.44210200862561\nFold 7: Acc 61.76470588235294, F1 50.55096164339862\nFold 8: Acc 58.69565217391305, F1 38.519258519258514\nAverage: 65.698%\nF1: 45.239%\nLoss: 0.727\n\n----------------------------------------------------------------------------\nFold 0: Acc 76.58%, F1 50.08%, Loss 0.52\nFold 1: Acc 71.30%, F1 46.66%, Loss 0.62\nFold 2: Acc 67.16%, F1 41.54%, Loss 0.66\nFold 3: Acc 70.45%, F1 44.65%, Loss 0.62\nFold 4: Acc 71.99%, F1 44.87%, Loss 0.64\nFold 5: Acc 78.57%, F1 64.84%, Loss 0.44\nFold 6: Acc 34.76%, F1 25.44%, Loss 1.29\nFold 7: Acc 61.76%, F1 50.55%, Loss 0.86\nFold 8: Acc 58.70%, F1 38.52%, Loss 0.89\n-----------------------------\n"
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                  BERT + W2V                                  #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_w2v_bert\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [50]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in bert_w2v_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_w2v_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+bert_w2v_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'BERT + W2V'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in bert_w2v_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### ROOT + POS + THREAD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "\n\n************************************************************\nSTARTING A CROSS-VALIDATION ... [100] epochs\n************************************************************\nTIME: 01/04/2021 10:16:23\nLAYER: [Sequential(\n  (0): Linear(in_features=101, out_features=24, bias=True)\n  (1): ELU(alpha=1.0)\n  (2): Dropout(p=0.5, inplace=False)\n  (3): Linear(in_features=24, out_features=6, bias=True)\n  (4): ELU(alpha=1.0)\n  (5): Dropout(p=0.5, inplace=False)\n  (6): Linear(in_features=6, out_features=1, bias=True)\n)]\n\nSTARTING TEST of 100 EPOCH\n\n> FOLD 1\nStarting epoch 50\nStarting epoch 100\n> FOLD 2\nStarting epoch 50\nStarting epoch 100\n> FOLD 3\nStarting epoch 50\nStarting epoch 100\n> FOLD 4\nStarting epoch 50\nStarting epoch 100\n> FOLD 5\nStarting epoch 50\nStarting epoch 100\n> FOLD 6\nStarting epoch 50\nStarting epoch 100\n> FOLD 7\nStarting epoch 50\nStarting epoch 100\n> FOLD 8\nStarting epoch 50\nStarting epoch 100\n> FOLD 9\nStarting epoch 50\nStarting epoch 100\n\n\n----------------------------------------------------------------------------\n>>>K-FOLD CROSS VALIDATION RESULTS FOR 9 FOLDS\n----------------------------------------------------------------------------\nFold 0: Acc 49.735449735449734, F1 32.3814270278917\nFold 1: Acc 51.70603674540683, F1 33.32614114766349\nFold 2: Acc 46.26865671641791, F1 30.746405927642595\nFold 3: Acc 49.10112359550562, F1 32.24378617637044\nFold 4: Acc 50.696150696150696, F1 33.222250267704865\nFold 5: Acc 57.14285714285714, F1 35.06493506493506\nFold 6: Acc 42.06008583690987, F1 29.116262850168425\nFold 7: Acc 48.319327731092436, F1 31.934074329032306\nFold 8: Acc 53.62318840579711, F1 35.24176306785002\nAverage: 49.850%\nF1: 32.586%\nLoss: 0.726\n\n----------------------------------------------------------------------------\nFold 0: Acc 49.74%, F1 32.38%, Loss 0.70\nFold 1: Acc 51.71%, F1 33.33%, Loss 0.77\nFold 2: Acc 46.27%, F1 30.75%, Loss 0.72\nFold 3: Acc 49.10%, F1 32.24%, Loss 0.70\nFold 4: Acc 50.70%, F1 33.22%, Loss 0.72\nFold 5: Acc 57.14%, F1 35.06%, Loss 0.69\nFold 6: Acc 42.06%, F1 29.12%, Loss 0.72\nFold 7: Acc 48.32%, F1 31.93%, Loss 0.83\nFold 8: Acc 53.62%, F1 35.24%, Loss 0.69\n-----------------------------\n"
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                              ROOT + POS + THREAD                             #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_root_thread\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [100]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in all_sparse_model().children()]}\")\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=all_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False, scaling=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "writeLog().writeWithoutCR(\"./Model/\"+all_sparse_model.__name__+\"_\"+\"log.txt\",f\"RESULT:\\n{epochs_diff(testing_results)}\")\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RPT'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in all_sparse_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RPT + W2V"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "\n\n************************************************************\nSTARTING A CROSS-VALIDATION ... [50] epochs\n************************************************************\nTIME: 01/04/2021 13:34:25\nLAYER: [Sequential(\n  (0): Linear(in_features=301, out_features=31, bias=True)\n  (1): ELU(alpha=1.0)\n  (2): Dropout(p=0.5, inplace=False)\n  (3): Linear(in_features=31, out_features=4, bias=True)\n  (4): ELU(alpha=1.0)\n  (5): Dropout(p=0.5, inplace=False)\n  (6): Linear(in_features=4, out_features=1, bias=True)\n)]\n\nSTARTING TEST of 50 EPOCH\n\n> FOLD 1\nStarting epoch 50\n> FOLD 2\nStarting epoch 50\n> FOLD 3\n"
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                 ALL Features                                 #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_total_w2v\n",
    "\n",
    "testing_epochs = [50]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+w2v_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+w2v_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+w2v_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in w2v_sparse_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=w2v_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "epochs_diff(testing_results)\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RPT + W2V'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in w2v_sparse_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = str(layers)\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### RPT + BERT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/x.notebook.stdout": "\n\n************************************************************\nSTARTING A CROSS-VALIDATION ... [5] epochs\n************************************************************\nTIME: 01/04/2021 13:24:27\nLAYER: [Sequential(\n  (0): Linear(in_features=869, out_features=80, bias=True)\n  (1): ELU(alpha=1.0)\n  (2): Dropout(p=0.5, inplace=False)\n  (3): Linear(in_features=80, out_features=16, bias=True)\n  (4): ELU(alpha=1.0)\n  (5): Dropout(p=0.5, inplace=False)\n  (6): Linear(in_features=16, out_features=1, bias=True)\n)]\n\nSTARTING TEST of 5 EPOCH\n\n> FOLD 1\n"
     },
     "output_type": "unknown"
    },
    {
     "data": {
      "application/x.notebook.error-traceback": {
       "ename": "KeyboardInterrupt",
       "evalue": "",
       "traceback": [
        "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
        "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
        "\u001b[0;32m<ipython-input-108-f74f3e1a5457>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     15\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     16\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mepoch\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtesting_epochs\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 17\u001b[0;31m     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mcv_process\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcriterion\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mmodelClass\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mbert_sparse_model\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mevents\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_event\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mtarget\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mall_y\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mepoch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     18\u001b[0m     \u001b[0mtesting_results\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresult\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m<ipython-input-95-f51df371c9d6>\u001b[0m in \u001b[0;36mcv_process\u001b[0;34m(dataset, criterion, modelClass, target, epochs, events, verbose, scaling)\u001b[0m\n\u001b[1;32m    124\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    125\u001b[0m             \u001b[0;31m# Iterate over the DataLoader for training data\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 126\u001b[0;31m             \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdata\u001b[0m \u001b[0;32min\u001b[0m \u001b[0menumerate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mtrain_dataloader\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    127\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    128\u001b[0m                 \u001b[0;31m# Get inputs\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m__next__\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    515\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_sampler_iter\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    516\u001b[0m                 \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 517\u001b[0;31m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    518\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_num_yielded\u001b[0m \u001b[0;34m+=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    519\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_kind\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0m_DatasetKind\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mIterable\u001b[0m \u001b[0;32mand\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/dataloader.py\u001b[0m in \u001b[0;36m_next_data\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    555\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_next_data\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    556\u001b[0m         \u001b[0mindex\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_next_index\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 557\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_dataset_fetcher\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfetch\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mindex\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# may raise StopIteration\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    558\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_pin_memory\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    559\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0m_utils\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpin_memory\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/fetch.py\u001b[0m in \u001b[0;36mfetch\u001b[0;34m(self, possibly_batched_index)\u001b[0m\n\u001b[1;32m     45\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     46\u001b[0m             \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdataset\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mpossibly_batched_index\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 47\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollate_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
        "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[0;34m(batch)\u001b[0m\n\u001b[1;32m     81\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     81\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mRuntimeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'each element in list of batch should be of equal size'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m         \u001b[0mtransposed\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mzip\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 83\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0mdefault_collate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msamples\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0msamples\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtransposed\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     84\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     85\u001b[0m     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdefault_collate_err_msg_format\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mformat\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0melem_type\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/torch/utils/data/_utils/collate.py\u001b[0m in \u001b[0;36mdefault_collate\u001b[0;34m(batch)\u001b[0m\n\u001b[1;32m     53\u001b[0m             \u001b[0mstorage\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_new_shared\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mnumel\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     54\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0melem\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mnew\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstorage\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 55\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mstack\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mout\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mout\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     56\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__module__\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'numpy'\u001b[0m \u001b[0;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'str_'\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m\\\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     57\u001b[0m             \u001b[0;32mand\u001b[0m \u001b[0melem_type\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m!=\u001b[0m \u001b[0;34m'string_'\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
        "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
       ]
      }
     },
     "output_type": "unknown"
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                                 ALL Features                                 #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "dataset = all_total_bert\n",
    "criterion = nn.BCEWithLogitsLoss()\n",
    "\n",
    "testing_epochs = [5]\n",
    "testing_epochs = testing_epochs\n",
    "\n",
    "testing_results = []\n",
    "\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"\\n\\n************************************************************\\nSTARTING A CROSS-VALIDATION ... {testing_epochs} epochs\\n************************************************************\")\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"TIME: {datetime.now().strftime('%d/%m/%Y %H:%M:%S')}\")\n",
    "writeLog().write(\"./Model/\"+bert_sparse_model.__name__+\"_\"+\"log.txt\",f\"LAYER: {[layer for layer in bert_sparse_model().children()]}\")\n",
    "\n",
    "\n",
    "for epoch in testing_epochs:\n",
    "    result = cv_process(dataset, criterion, modelClass=bert_sparse_model, events=all_event, target=all_y, epochs=epoch, verbose=False)\n",
    "    testing_results.append(result)\n",
    "\n",
    "epochs_diff(testing_results)\n",
    "result_df = pd.DataFrame.from_dict(testing_results[0], orient='index', columns=['Acc', 'F1', 'Loss', 'Train_size','Test_size'])\n",
    "result_df['Features'] = 'RPT + BERT'\n",
    "result_df['CV'] = 9\n",
    "layers = [module for module in bert_sparse_model().modules() if not isinstance(module, nn.Sequential)]\n",
    "layers = [layer.in_features for layer in layers[1:] if str(layer).startswith(\"Linear\")]\n",
    "result_df['layers'] = layers\n",
    "\n",
    "result_list.append(result_df)\n",
    "CV_result_df = pd.DataFrame(result_df[['Acc','F1', 'Loss', 'CV']].mean(), columns=[result_df.Features[0]]).T\n",
    "CV_result_df['layers'] = result_df['layers'][0]\n",
    "\n",
    "result_mean_list.append(CV_result_df)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## RESULT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] No such file or directory: './data/final_result_5.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-106-a3237f25007d>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m# final_result_5 = pd.concat(result_list_5)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[0;31m# final_result_5.to_csv('./data/final_result_5.csv')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 7\u001b[0;31m \u001b[0mfinal_result_5\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'./data/final_result_5.csv'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mindex_col\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'Unnamed: 0'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      8\u001b[0m \u001b[0;31m# final_cv_5 = pd.concat(result_mean_list_5)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;31m# final_cv_5.to_csv('./data/final_cv_5.csv')\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mread_csv\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision, storage_options)\u001b[0m\n\u001b[1;32m    608\u001b[0m     \u001b[0mkwds\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mupdate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mkwds_defaults\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    609\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 610\u001b[0;31m     \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    611\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    612\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    460\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    461\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 462\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    463\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    464\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    817\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    818\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 819\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    820\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    821\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1048\u001b[0m             )\n\u001b[1;32m   1049\u001b[0m         \u001b[0;31m# error: Too many arguments for \"ParserBase\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1050\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mmapping\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# type: ignore[call-arg]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1051\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1052\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_failover_to_python\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1865\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1866\u001b[0m         \u001b[0;31m# open handles\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1867\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_open_handles\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1868\u001b[0m         \u001b[0;32massert\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhandles\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1869\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mkey\u001b[0m \u001b[0;32min\u001b[0m \u001b[0;34m(\u001b[0m\u001b[0;34m\"storage_options\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"encoding\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"memory_map\"\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"compression\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_open_handles\u001b[0;34m(self, src, kwds)\u001b[0m\n\u001b[1;32m   1360\u001b[0m         \u001b[0mLet\u001b[0m \u001b[0mthe\u001b[0m \u001b[0mreaders\u001b[0m \u001b[0mopen\u001b[0m \u001b[0mIOHanldes\u001b[0m \u001b[0mafter\u001b[0m \u001b[0mthey\u001b[0m \u001b[0mare\u001b[0m \u001b[0mdone\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mtheir\u001b[0m \u001b[0mpotential\u001b[0m \u001b[0mraises\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1361\u001b[0m         \"\"\"\n\u001b[0;32m-> 1362\u001b[0;31m         self.handles = get_handle(\n\u001b[0m\u001b[1;32m   1363\u001b[0m             \u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1364\u001b[0m             \u001b[0;34m\"r\"\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/envs/rosetta/lib/python3.8/site-packages/pandas/io/common.py\u001b[0m in \u001b[0;36mget_handle\u001b[0;34m(path_or_buf, mode, encoding, compression, memory_map, is_text, errors, storage_options)\u001b[0m\n\u001b[1;32m    640\u001b[0m                 \u001b[0merrors\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m\"replace\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    641\u001b[0m             \u001b[0;31m# Encoding\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 642\u001b[0;31m             handle = open(\n\u001b[0m\u001b[1;32m    643\u001b[0m                 \u001b[0mhandle\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    644\u001b[0m                 \u001b[0mioargs\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmode\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] No such file or directory: './data/final_result_5.csv'"
     ]
    }
   ],
   "source": [
    "# ---------------------------------------------------------------------------- #\n",
    "#                               RESULT OF BATCH 5                              #\n",
    "# ---------------------------------------------------------------------------- #\n",
    "\n",
    "# final_result_5 = pd.concat(result_list_5)\n",
    "# final_result_5.to_csv('./data/final_result_5.csv')\n",
    "final_result_5 = pd.read_csv('./data/final_result_5.csv', index_col=['Unnamed: 0'])\n",
    "# final_cv_5 = pd.concat(result_mean_list_5)\n",
    "# final_cv_5.to_csv('./data/final_cv_5.csv')\n",
    "final_cv_5 = pd.read_csv('./data/final_cv_5.csv', index_col=['Unnamed: 0'])\n",
    "final_cv_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "final_result_5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 109,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Acc</th>\n      <th>F1</th>\n      <th>Loss</th>\n      <th>CV</th>\n      <th>layers</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>W2V</th>\n      <td>65.783691</td>\n      <td>44.733944</td>\n      <td>0.664950</td>\n      <td>9.0</td>\n      <td>[200, 36, 8]</td>\n    </tr>\n    <tr>\n      <th>BERT + W2V</th>\n      <td>66.369984</td>\n      <td>45.486365</td>\n      <td>0.696974</td>\n      <td>9.0</td>\n      <td>[968, 80, 12]</td>\n    </tr>\n    <tr>\n      <th>BERT + W2V</th>\n      <td>64.828757</td>\n      <td>43.531413</td>\n      <td>0.686351</td>\n      <td>9.0</td>\n      <td>[968, 80, 12, 4]</td>\n    </tr>\n    <tr>\n      <th>BERT + W2V</th>\n      <td>65.697599</td>\n      <td>45.238628</td>\n      <td>0.726546</td>\n      <td>9.0</td>\n      <td>[968, 64, 12]</td>\n    </tr>\n    <tr>\n      <th>RPT + W2V</th>\n      <td>52.371971</td>\n      <td>33.838737</td>\n      <td>1.481236</td>\n      <td>9.0</td>\n      <td>[301, 48, 8]</td>\n    </tr>\n    <tr>\n      <th>RPT + W2V</th>\n      <td>54.028256</td>\n      <td>34.706857</td>\n      <td>1.465343</td>\n      <td>9.0</td>\n      <td>[301, 52, 12]</td>\n    </tr>\n    <tr>\n      <th>RPT + W2V</th>\n      <td>49.238179</td>\n      <td>32.481797</td>\n      <td>1.843140</td>\n      <td>9.0</td>\n      <td>[301, 52]</td>\n    </tr>\n    <tr>\n      <th>RPT + W2V</th>\n      <td>48.116611</td>\n      <td>31.456886</td>\n      <td>0.859566</td>\n      <td>9.0</td>\n      <td>[301, 30, 5]</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "                  Acc         F1      Loss   CV            layers\nW2V         65.783691  44.733944  0.664950  9.0      [200, 36, 8]\nBERT + W2V  66.369984  45.486365  0.696974  9.0     [968, 80, 12]\nBERT + W2V  64.828757  43.531413  0.686351  9.0  [968, 80, 12, 4]\nBERT + W2V  65.697599  45.238628  0.726546  9.0     [968, 64, 12]\nRPT + W2V   52.371971  33.838737  1.481236  9.0      [301, 48, 8]\nRPT + W2V   54.028256  34.706857  1.465343  9.0     [301, 52, 12]\nRPT + W2V   49.238179  32.481797  1.843140  9.0         [301, 52]\nRPT + W2V   48.116611  31.456886  0.859566  9.0      [301, 30, 5]"
     },
     "execution_count": 109,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# final_result = pd.concat(result_list)\n",
    "# final_result.to_csv('./data/final_result.csv')\n",
    "final_result = pd.read_csv('./data/final_result.csv', index_col=['Unnamed: 0'])\n",
    "# final_cv = pd.concat(result_mean_list)\n",
    "# final_cv.to_csv('./data/final_cv.csv')\n",
    "final_cv = pd.read_csv('./data/final_cv.csv', index_col=['Unnamed: 0'])\n",
    "final_cv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 110,
   "metadata": {},
   "outputs": [],
   "source": [
    "# final_result"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.8.8 64-bit (conda)",
   "name": "python388jvsc74a57bd0b3e779290c3971bcb91630500d66c3c3cecd721489b4b277b4ac4fef67ef773c"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "metadata": {
   "interpreter": {
    "hash": "b3e779290c3971bcb91630500d66c3c3cecd721489b4b277b4ac4fef67ef773c"
   }
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}