main-notebook.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:11.791574Z",
     "iopub.status.busy": "2020-11-26T03:05:11.790719Z",
     "iopub.status.idle": "2020-11-26T03:05:21.694280Z",
     "shell.execute_reply": "2020-11-26T03:05:21.693216Z"
    },
    "papermill": {
     "duration": 9.961318,
     "end_time": "2020-11-26T03:05:21.694417",
     "exception": false,
     "start_time": "2020-11-26T03:05:11.733099",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in links: /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl\r\n",
      "Processing /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl\r\n",
      "Requirement already satisfied: numpy<2.0,>=1.17 in /opt/conda/lib/python3.7/site-packages (from pytorch-tabnet) (1.18.5)\r\n",
      "Requirement already satisfied: scikit_learn>0.21 in /opt/conda/lib/python3.7/site-packages (from pytorch-tabnet) (0.23.2)\r\n",
      "Requirement already satisfied: scipy>1.4 in /opt/conda/lib/python3.7/site-packages (from pytorch-tabnet) (1.4.1)\r\n",
      "Requirement already satisfied: torch<2.0,>=1.2 in /opt/conda/lib/python3.7/site-packages (from pytorch-tabnet) (1.6.0)\r\n",
      "Requirement already satisfied: tqdm<5.0,>=4.36 in /opt/conda/lib/python3.7/site-packages (from pytorch-tabnet) (4.45.0)\r\n",
      "Requirement already satisfied: joblib>=0.11 in /opt/conda/lib/python3.7/site-packages (from scikit_learn>0.21->pytorch-tabnet) (0.14.1)\r\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /opt/conda/lib/python3.7/site-packages (from scikit_learn>0.21->pytorch-tabnet) (2.1.0)\r\n",
      "Requirement already satisfied: future in /opt/conda/lib/python3.7/site-packages (from torch<2.0,>=1.2->pytorch-tabnet) (0.18.2)\r\n",
      "Installing collected packages: pytorch-tabnet\r\n",
      "Successfully installed pytorch-tabnet-2.0.0\r\n"
     ]
    }
   ],
   "source": [
    "!pip install --no-index --find-links /kaggle/input/pytorchtabnet/pytorch_tabnet-2.0.0-py3-none-any.whl pytorch-tabnet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:21.808946Z",
     "iopub.status.busy": "2020-11-26T03:05:21.805904Z",
     "iopub.status.idle": "2020-11-26T03:05:30.852293Z",
     "shell.execute_reply": "2020-11-26T03:05:30.853909Z"
    },
    "papermill": {
     "duration": 9.108248,
     "end_time": "2020-11-26T03:05:30.854176",
     "exception": false,
     "start_time": "2020-11-26T03:05:21.745928",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/tensorflow_addons/utils/ensure_tf_install.py:68: UserWarning: Tensorflow Addons supports using Python ops for all Tensorflow versions above or equal to 2.2.0 and strictly below 2.3.0 (nightly versions are not supported). \n",
      " The versions of TensorFlow you are currently using is 2.3.1 and is not supported. \n",
      "Some things might work, some things might not.\n",
      "If you were to encounter a bug, do not file an issue.\n",
      "If you want to make sure you're using a tested and supported configuration, either change the TensorFlow version or the TensorFlow Addons's version. \n",
      "You can find the compatibility matrix in TensorFlow Addon's readme:\n",
      "https://github.com/tensorflow/addons\n",
      "  UserWarning,\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import QuantileTransformer\n",
    "from sklearn.decomposition import PCA\n",
    "from sklearn.cluster import KMeans\n",
    "import tensorflow as tf\n",
    "import tensorflow_addons as tfa\n",
    "import sys\n",
    "sys.path.append('../input/iterative-stratification/iterative-stratification-master')\n",
    "from iterstrat.ml_stratifiers import MultilabelStratifiedKFold\n",
    "from pytorch_tabnet.tab_model import TabNetRegressor\n",
    "import torch.optim as optim\n",
    "from torch.optim.lr_scheduler import ReduceLROnPlateau\n",
    "from torch.nn import functional as F"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:31.029374Z",
     "iopub.status.busy": "2020-11-26T03:05:31.028269Z",
     "iopub.status.idle": "2020-11-26T03:05:39.279698Z",
     "shell.execute_reply": "2020-11-26T03:05:39.278963Z"
    },
    "papermill": {
     "duration": 8.341031,
     "end_time": "2020-11-26T03:05:39.279823",
     "exception": false,
     "start_time": "2020-11-26T03:05:30.938792",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "x_train = pd.read_csv('../input/lish-moa/train_features.csv')\n",
    "x_test = pd.read_csv('../input/lish-moa/test_features.csv')\n",
    "y_train = pd.read_csv('../input/lish-moa/train_targets_scored.csv')\n",
    "y_train_non = pd.read_csv('../input/lish-moa/train_targets_nonscored.csv')\n",
    "submission = pd.read_csv('../input/lish-moa/sample_submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:39.394071Z",
     "iopub.status.busy": "2020-11-26T03:05:39.392672Z",
     "iopub.status.idle": "2020-11-26T03:05:39.397358Z",
     "shell.execute_reply": "2020-11-26T03:05:39.397967Z"
    },
    "papermill": {
     "duration": 0.066096,
     "end_time": "2020-11-26T03:05:39.398122",
     "exception": false,
     "start_time": "2020-11-26T03:05:39.332026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(23814, 403)"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_train_non.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:39.511952Z",
     "iopub.status.busy": "2020-11-26T03:05:39.511234Z",
     "iopub.status.idle": "2020-11-26T03:05:39.581789Z",
     "shell.execute_reply": "2020-11-26T03:05:39.581155Z"
    },
    "papermill": {
     "duration": 0.131735,
     "end_time": "2020-11-26T03:05:39.581910",
     "exception": false,
     "start_time": "2020-11-26T03:05:39.450175",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "trt_cp_index=x_train[x_train['cp_type']=='trt_cp'].index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:39.728284Z",
     "iopub.status.busy": "2020-11-26T03:05:39.726186Z",
     "iopub.status.idle": "2020-11-26T03:05:39.734706Z",
     "shell.execute_reply": "2020-11-26T03:05:39.734014Z"
    },
    "papermill": {
     "duration": 0.087636,
     "end_time": "2020-11-26T03:05:39.734828",
     "exception": false,
     "start_time": "2020-11-26T03:05:39.647192",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "x_test_checkpoint=x_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:39.863665Z",
     "iopub.status.busy": "2020-11-26T03:05:39.857462Z",
     "iopub.status.idle": "2020-11-26T03:05:39.883330Z",
     "shell.execute_reply": "2020-11-26T03:05:39.883985Z"
    },
    "papermill": {
     "duration": 0.097206,
     "end_time": "2020-11-26T03:05:39.884149",
     "exception": false,
     "start_time": "2020-11-26T03:05:39.786943",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sig_id</th>\n",
       "      <th>cp_type</th>\n",
       "      <th>cp_time</th>\n",
       "      <th>cp_dose</th>\n",
       "      <th>g-0</th>\n",
       "      <th>g-1</th>\n",
       "      <th>g-2</th>\n",
       "      <th>g-3</th>\n",
       "      <th>g-4</th>\n",
       "      <th>g-5</th>\n",
       "      <th>...</th>\n",
       "      <th>c-90</th>\n",
       "      <th>c-91</th>\n",
       "      <th>c-92</th>\n",
       "      <th>c-93</th>\n",
       "      <th>c-94</th>\n",
       "      <th>c-95</th>\n",
       "      <th>c-96</th>\n",
       "      <th>c-97</th>\n",
       "      <th>c-98</th>\n",
       "      <th>c-99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_0004d9e33</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>24</td>\n",
       "      <td>D1</td>\n",
       "      <td>-0.5458</td>\n",
       "      <td>0.1306</td>\n",
       "      <td>-0.5135</td>\n",
       "      <td>0.4408</td>\n",
       "      <td>1.5500</td>\n",
       "      <td>-0.1644</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0981</td>\n",
       "      <td>0.7978</td>\n",
       "      <td>-0.1430</td>\n",
       "      <td>-0.2067</td>\n",
       "      <td>-0.2303</td>\n",
       "      <td>-0.1193</td>\n",
       "      <td>0.0210</td>\n",
       "      <td>-0.0502</td>\n",
       "      <td>0.1510</td>\n",
       "      <td>-0.7750</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_001897cda</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>72</td>\n",
       "      <td>D1</td>\n",
       "      <td>-0.1829</td>\n",
       "      <td>0.2320</td>\n",
       "      <td>1.2080</td>\n",
       "      <td>-0.4522</td>\n",
       "      <td>-0.3652</td>\n",
       "      <td>-0.3319</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.1190</td>\n",
       "      <td>-0.1852</td>\n",
       "      <td>-1.0310</td>\n",
       "      <td>-1.3670</td>\n",
       "      <td>-0.3690</td>\n",
       "      <td>-0.5382</td>\n",
       "      <td>0.0359</td>\n",
       "      <td>-0.4764</td>\n",
       "      <td>-1.3810</td>\n",
       "      <td>-0.7300</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_002429b5b</td>\n",
       "      <td>ctl_vehicle</td>\n",
       "      <td>24</td>\n",
       "      <td>D1</td>\n",
       "      <td>0.1852</td>\n",
       "      <td>-0.1404</td>\n",
       "      <td>-0.3911</td>\n",
       "      <td>0.1310</td>\n",
       "      <td>-1.4380</td>\n",
       "      <td>0.2455</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.2261</td>\n",
       "      <td>0.3370</td>\n",
       "      <td>-1.3840</td>\n",
       "      <td>0.8604</td>\n",
       "      <td>-1.9530</td>\n",
       "      <td>-1.0140</td>\n",
       "      <td>0.8662</td>\n",
       "      <td>1.0160</td>\n",
       "      <td>0.4924</td>\n",
       "      <td>-0.1942</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_00276f245</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>24</td>\n",
       "      <td>D2</td>\n",
       "      <td>0.4828</td>\n",
       "      <td>0.1955</td>\n",
       "      <td>0.3825</td>\n",
       "      <td>0.4244</td>\n",
       "      <td>-0.5855</td>\n",
       "      <td>-1.2020</td>\n",
       "      <td>...</td>\n",
       "      <td>0.1260</td>\n",
       "      <td>0.1570</td>\n",
       "      <td>-0.1784</td>\n",
       "      <td>-1.1200</td>\n",
       "      <td>-0.4325</td>\n",
       "      <td>-0.9005</td>\n",
       "      <td>0.8131</td>\n",
       "      <td>-0.1305</td>\n",
       "      <td>0.5645</td>\n",
       "      <td>-0.5809</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_0027f1083</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>48</td>\n",
       "      <td>D1</td>\n",
       "      <td>-0.3979</td>\n",
       "      <td>-1.2680</td>\n",
       "      <td>1.9130</td>\n",
       "      <td>0.2057</td>\n",
       "      <td>-0.5864</td>\n",
       "      <td>-0.0166</td>\n",
       "      <td>...</td>\n",
       "      <td>0.4965</td>\n",
       "      <td>0.7578</td>\n",
       "      <td>-0.1580</td>\n",
       "      <td>1.0510</td>\n",
       "      <td>0.5742</td>\n",
       "      <td>1.0900</td>\n",
       "      <td>-0.2962</td>\n",
       "      <td>-0.5313</td>\n",
       "      <td>0.9931</td>\n",
       "      <td>1.8380</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 876 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         sig_id      cp_type  cp_time cp_dose     g-0     g-1     g-2     g-3  \\\n",
       "0  id_0004d9e33       trt_cp       24      D1 -0.5458  0.1306 -0.5135  0.4408   \n",
       "1  id_001897cda       trt_cp       72      D1 -0.1829  0.2320  1.2080 -0.4522   \n",
       "2  id_002429b5b  ctl_vehicle       24      D1  0.1852 -0.1404 -0.3911  0.1310   \n",
       "3  id_00276f245       trt_cp       24      D2  0.4828  0.1955  0.3825  0.4244   \n",
       "4  id_0027f1083       trt_cp       48      D1 -0.3979 -1.2680  1.9130  0.2057   \n",
       "\n",
       "      g-4     g-5  ...    c-90    c-91    c-92    c-93    c-94    c-95  \\\n",
       "0  1.5500 -0.1644  ...  0.0981  0.7978 -0.1430 -0.2067 -0.2303 -0.1193   \n",
       "1 -0.3652 -0.3319  ... -0.1190 -0.1852 -1.0310 -1.3670 -0.3690 -0.5382   \n",
       "2 -1.4380  0.2455  ... -0.2261  0.3370 -1.3840  0.8604 -1.9530 -1.0140   \n",
       "3 -0.5855 -1.2020  ...  0.1260  0.1570 -0.1784 -1.1200 -0.4325 -0.9005   \n",
       "4 -0.5864 -0.0166  ...  0.4965  0.7578 -0.1580  1.0510  0.5742  1.0900   \n",
       "\n",
       "     c-96    c-97    c-98    c-99  \n",
       "0  0.0210 -0.0502  0.1510 -0.7750  \n",
       "1  0.0359 -0.4764 -1.3810 -0.7300  \n",
       "2  0.8662  1.0160  0.4924 -0.1942  \n",
       "3  0.8131 -0.1305  0.5645 -0.5809  \n",
       "4 -0.2962 -0.5313  0.9931  1.8380  \n",
       "\n",
       "[5 rows x 876 columns]"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test_checkpoint.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:40.028781Z",
     "iopub.status.busy": "2020-11-26T03:05:40.004473Z",
     "iopub.status.idle": "2020-11-26T03:05:40.036659Z",
     "shell.execute_reply": "2020-11-26T03:05:40.035912Z"
    },
    "papermill": {
     "duration": 0.093426,
     "end_time": "2020-11-26T03:05:40.036791",
     "exception": false,
     "start_time": "2020-11-26T03:05:39.943365",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sig_id</th>\n",
       "      <th>cp_type</th>\n",
       "      <th>cp_time</th>\n",
       "      <th>cp_dose</th>\n",
       "      <th>g-0</th>\n",
       "      <th>g-1</th>\n",
       "      <th>g-2</th>\n",
       "      <th>g-3</th>\n",
       "      <th>g-4</th>\n",
       "      <th>g-5</th>\n",
       "      <th>...</th>\n",
       "      <th>c-90</th>\n",
       "      <th>c-91</th>\n",
       "      <th>c-92</th>\n",
       "      <th>c-93</th>\n",
       "      <th>c-94</th>\n",
       "      <th>c-95</th>\n",
       "      <th>c-96</th>\n",
       "      <th>c-97</th>\n",
       "      <th>c-98</th>\n",
       "      <th>c-99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_000644bb2</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>24</td>\n",
       "      <td>D1</td>\n",
       "      <td>1.0620</td>\n",
       "      <td>0.5577</td>\n",
       "      <td>-0.2479</td>\n",
       "      <td>-0.6208</td>\n",
       "      <td>-0.1944</td>\n",
       "      <td>-1.0120</td>\n",
       "      <td>...</td>\n",
       "      <td>0.2862</td>\n",
       "      <td>0.2584</td>\n",
       "      <td>0.8076</td>\n",
       "      <td>0.5523</td>\n",
       "      <td>-0.1912</td>\n",
       "      <td>0.6584</td>\n",
       "      <td>-0.3981</td>\n",
       "      <td>0.2139</td>\n",
       "      <td>0.3801</td>\n",
       "      <td>0.4176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_000779bfc</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>72</td>\n",
       "      <td>D1</td>\n",
       "      <td>0.0743</td>\n",
       "      <td>0.4087</td>\n",
       "      <td>0.2991</td>\n",
       "      <td>0.0604</td>\n",
       "      <td>1.0190</td>\n",
       "      <td>0.5207</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.4265</td>\n",
       "      <td>0.7543</td>\n",
       "      <td>0.4708</td>\n",
       "      <td>0.0230</td>\n",
       "      <td>0.2957</td>\n",
       "      <td>0.4899</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1241</td>\n",
       "      <td>0.6077</td>\n",
       "      <td>0.7371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_000a6266a</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>48</td>\n",
       "      <td>D1</td>\n",
       "      <td>0.6280</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>1.5540</td>\n",
       "      <td>-0.0764</td>\n",
       "      <td>-0.0323</td>\n",
       "      <td>1.2390</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.7250</td>\n",
       "      <td>-0.6297</td>\n",
       "      <td>0.6103</td>\n",
       "      <td>0.0223</td>\n",
       "      <td>-1.3240</td>\n",
       "      <td>-0.3174</td>\n",
       "      <td>-0.6417</td>\n",
       "      <td>-0.2187</td>\n",
       "      <td>-1.4080</td>\n",
       "      <td>0.6931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_0015fd391</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>48</td>\n",
       "      <td>D1</td>\n",
       "      <td>-0.5138</td>\n",
       "      <td>-0.2491</td>\n",
       "      <td>-0.2656</td>\n",
       "      <td>0.5288</td>\n",
       "      <td>4.0620</td>\n",
       "      <td>-0.8095</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.0990</td>\n",
       "      <td>-0.6441</td>\n",
       "      <td>-5.6300</td>\n",
       "      <td>-1.3780</td>\n",
       "      <td>-0.8632</td>\n",
       "      <td>-1.2880</td>\n",
       "      <td>-1.6210</td>\n",
       "      <td>-0.8784</td>\n",
       "      <td>-0.3876</td>\n",
       "      <td>-0.8154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_001626bd3</td>\n",
       "      <td>trt_cp</td>\n",
       "      <td>72</td>\n",
       "      <td>D2</td>\n",
       "      <td>-0.3254</td>\n",
       "      <td>-0.4009</td>\n",
       "      <td>0.9700</td>\n",
       "      <td>0.6919</td>\n",
       "      <td>1.4180</td>\n",
       "      <td>-0.8244</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.6670</td>\n",
       "      <td>1.0690</td>\n",
       "      <td>0.5523</td>\n",
       "      <td>-0.3031</td>\n",
       "      <td>0.1094</td>\n",
       "      <td>0.2885</td>\n",
       "      <td>-0.3786</td>\n",
       "      <td>0.7125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 876 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         sig_id cp_type  cp_time cp_dose     g-0     g-1     g-2     g-3  \\\n",
       "0  id_000644bb2  trt_cp       24      D1  1.0620  0.5577 -0.2479 -0.6208   \n",
       "1  id_000779bfc  trt_cp       72      D1  0.0743  0.4087  0.2991  0.0604   \n",
       "2  id_000a6266a  trt_cp       48      D1  0.6280  0.5817  1.5540 -0.0764   \n",
       "3  id_0015fd391  trt_cp       48      D1 -0.5138 -0.2491 -0.2656  0.5288   \n",
       "4  id_001626bd3  trt_cp       72      D2 -0.3254 -0.4009  0.9700  0.6919   \n",
       "\n",
       "      g-4     g-5  ...    c-90    c-91    c-92    c-93    c-94    c-95  \\\n",
       "0 -0.1944 -1.0120  ...  0.2862  0.2584  0.8076  0.5523 -0.1912  0.6584   \n",
       "1  1.0190  0.5207  ... -0.4265  0.7543  0.4708  0.0230  0.2957  0.4899   \n",
       "2 -0.0323  1.2390  ... -0.7250 -0.6297  0.6103  0.0223 -1.3240 -0.3174   \n",
       "3  4.0620 -0.8095  ... -2.0990 -0.6441 -5.6300 -1.3780 -0.8632 -1.2880   \n",
       "4  1.4180 -0.8244  ...  0.0042  0.0048  0.6670  1.0690  0.5523 -0.3031   \n",
       "\n",
       "     c-96    c-97    c-98    c-99  \n",
       "0 -0.3981  0.2139  0.3801  0.4176  \n",
       "1  0.1522  0.1241  0.6077  0.7371  \n",
       "2 -0.6417 -0.2187 -1.4080  0.6931  \n",
       "3 -1.6210 -0.8784 -0.3876 -0.8154  \n",
       "4  0.1094  0.2885 -0.3786  0.7125  \n",
       "\n",
       "[5 rows x 876 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:40.239407Z",
     "iopub.status.busy": "2020-11-26T03:05:40.211370Z",
     "iopub.status.idle": "2020-11-26T03:05:40.269887Z",
     "shell.execute_reply": "2020-11-26T03:05:40.270528Z"
    },
    "papermill": {
     "duration": 0.173595,
     "end_time": "2020-11-26T03:05:40.270715",
     "exception": false,
     "start_time": "2020-11-26T03:05:40.097120",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cp_type</th>\n",
       "      <th>cp_time</th>\n",
       "      <th>cp_dose</th>\n",
       "      <th>g-0</th>\n",
       "      <th>g-1</th>\n",
       "      <th>g-2</th>\n",
       "      <th>g-3</th>\n",
       "      <th>g-4</th>\n",
       "      <th>g-5</th>\n",
       "      <th>g-6</th>\n",
       "      <th>...</th>\n",
       "      <th>c-90</th>\n",
       "      <th>c-91</th>\n",
       "      <th>c-92</th>\n",
       "      <th>c-93</th>\n",
       "      <th>c-94</th>\n",
       "      <th>c-95</th>\n",
       "      <th>c-96</th>\n",
       "      <th>c-97</th>\n",
       "      <th>c-98</th>\n",
       "      <th>c-99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>trt_cp</td>\n",
       "      <td>24</td>\n",
       "      <td>D1</td>\n",
       "      <td>1.0620</td>\n",
       "      <td>0.5577</td>\n",
       "      <td>-0.2479</td>\n",
       "      <td>-0.6208</td>\n",
       "      <td>-0.1944</td>\n",
       "      <td>-1.0120</td>\n",
       "      <td>-1.0220</td>\n",
       "      <td>...</td>\n",
       "      <td>0.2862</td>\n",
       "      <td>0.2584</td>\n",
       "      <td>0.8076</td>\n",
       "      <td>0.5523</td>\n",
       "      <td>-0.1912</td>\n",
       "      <td>0.6584</td>\n",
       "      <td>-0.3981</td>\n",
       "      <td>0.2139</td>\n",
       "      <td>0.3801</td>\n",
       "      <td>0.4176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>trt_cp</td>\n",
       "      <td>72</td>\n",
       "      <td>D1</td>\n",
       "      <td>0.0743</td>\n",
       "      <td>0.4087</td>\n",
       "      <td>0.2991</td>\n",
       "      <td>0.0604</td>\n",
       "      <td>1.0190</td>\n",
       "      <td>0.5207</td>\n",
       "      <td>0.2341</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.4265</td>\n",
       "      <td>0.7543</td>\n",
       "      <td>0.4708</td>\n",
       "      <td>0.0230</td>\n",
       "      <td>0.2957</td>\n",
       "      <td>0.4899</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1241</td>\n",
       "      <td>0.6077</td>\n",
       "      <td>0.7371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>trt_cp</td>\n",
       "      <td>48</td>\n",
       "      <td>D1</td>\n",
       "      <td>0.6280</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>1.5540</td>\n",
       "      <td>-0.0764</td>\n",
       "      <td>-0.0323</td>\n",
       "      <td>1.2390</td>\n",
       "      <td>0.1715</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.7250</td>\n",
       "      <td>-0.6297</td>\n",
       "      <td>0.6103</td>\n",
       "      <td>0.0223</td>\n",
       "      <td>-1.3240</td>\n",
       "      <td>-0.3174</td>\n",
       "      <td>-0.6417</td>\n",
       "      <td>-0.2187</td>\n",
       "      <td>-1.4080</td>\n",
       "      <td>0.6931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>trt_cp</td>\n",
       "      <td>48</td>\n",
       "      <td>D1</td>\n",
       "      <td>-0.5138</td>\n",
       "      <td>-0.2491</td>\n",
       "      <td>-0.2656</td>\n",
       "      <td>0.5288</td>\n",
       "      <td>4.0620</td>\n",
       "      <td>-0.8095</td>\n",
       "      <td>-1.9590</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.0990</td>\n",
       "      <td>-0.6441</td>\n",
       "      <td>-5.6300</td>\n",
       "      <td>-1.3780</td>\n",
       "      <td>-0.8632</td>\n",
       "      <td>-1.2880</td>\n",
       "      <td>-1.6210</td>\n",
       "      <td>-0.8784</td>\n",
       "      <td>-0.3876</td>\n",
       "      <td>-0.8154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>trt_cp</td>\n",
       "      <td>72</td>\n",
       "      <td>D2</td>\n",
       "      <td>-0.3254</td>\n",
       "      <td>-0.4009</td>\n",
       "      <td>0.9700</td>\n",
       "      <td>0.6919</td>\n",
       "      <td>1.4180</td>\n",
       "      <td>-0.8244</td>\n",
       "      <td>-0.2800</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.6670</td>\n",
       "      <td>1.0690</td>\n",
       "      <td>0.5523</td>\n",
       "      <td>-0.3031</td>\n",
       "      <td>0.1094</td>\n",
       "      <td>0.2885</td>\n",
       "      <td>-0.3786</td>\n",
       "      <td>0.7125</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 875 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "  cp_type  cp_time cp_dose     g-0     g-1     g-2     g-3     g-4     g-5  \\\n",
       "0  trt_cp       24      D1  1.0620  0.5577 -0.2479 -0.6208 -0.1944 -1.0120   \n",
       "1  trt_cp       72      D1  0.0743  0.4087  0.2991  0.0604  1.0190  0.5207   \n",
       "2  trt_cp       48      D1  0.6280  0.5817  1.5540 -0.0764 -0.0323  1.2390   \n",
       "3  trt_cp       48      D1 -0.5138 -0.2491 -0.2656  0.5288  4.0620 -0.8095   \n",
       "4  trt_cp       72      D2 -0.3254 -0.4009  0.9700  0.6919  1.4180 -0.8244   \n",
       "\n",
       "      g-6  ...    c-90    c-91    c-92    c-93    c-94    c-95    c-96  \\\n",
       "0 -1.0220  ...  0.2862  0.2584  0.8076  0.5523 -0.1912  0.6584 -0.3981   \n",
       "1  0.2341  ... -0.4265  0.7543  0.4708  0.0230  0.2957  0.4899  0.1522   \n",
       "2  0.1715  ... -0.7250 -0.6297  0.6103  0.0223 -1.3240 -0.3174 -0.6417   \n",
       "3 -1.9590  ... -2.0990 -0.6441 -5.6300 -1.3780 -0.8632 -1.2880 -1.6210   \n",
       "4 -0.2800  ...  0.0042  0.0048  0.6670  1.0690  0.5523 -0.3031  0.1094   \n",
       "\n",
       "     c-97    c-98    c-99  \n",
       "0  0.2139  0.3801  0.4176  \n",
       "1  0.1241  0.6077  0.7371  \n",
       "2 -0.2187 -1.4080  0.6931  \n",
       "3 -0.8784 -0.3876 -0.8154  \n",
       "4  0.2885 -0.3786  0.7125  \n",
       "\n",
       "[5 rows x 875 columns]"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train=x_train.drop(['sig_id'],1)\n",
    "x_test=x_test.drop(['sig_id'],1)\n",
    "y_train=y_train.drop(['sig_id'],1)\n",
    "x_train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:40.453240Z",
     "iopub.status.busy": "2020-11-26T03:05:40.451575Z",
     "iopub.status.idle": "2020-11-26T03:05:40.467345Z",
     "shell.execute_reply": "2020-11-26T03:05:40.466675Z"
    },
    "papermill": {
     "duration": 0.135378,
     "end_time": "2020-11-26T03:05:40.467484",
     "exception": false,
     "start_time": "2020-11-26T03:05:40.332106",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "x_train=x_train.drop(['cp_type'],1)\n",
    "x_test=x_test.drop(['cp_type'],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:40.598868Z",
     "iopub.status.busy": "2020-11-26T03:05:40.597723Z",
     "iopub.status.idle": "2020-11-26T03:05:40.608443Z",
     "shell.execute_reply": "2020-11-26T03:05:40.607810Z"
    },
    "papermill": {
     "duration": 0.0803,
     "end_time": "2020-11-26T03:05:40.608591",
     "exception": false,
     "start_time": "2020-11-26T03:05:40.528291",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 2, 1])"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train['cp_time'] = x_train['cp_time'].map({24:0,48:1,72:2})\n",
    "x_train.cp_time.unique()\n",
    "x_test['cp_time'] = x_test['cp_time'].map({24:0,48:1,72:2})\n",
    "x_test.cp_time.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:40.744492Z",
     "iopub.status.busy": "2020-11-26T03:05:40.743433Z",
     "iopub.status.idle": "2020-11-26T03:05:40.753415Z",
     "shell.execute_reply": "2020-11-26T03:05:40.752860Z"
    },
    "papermill": {
     "duration": 0.083814,
     "end_time": "2020-11-26T03:05:40.753534",
     "exception": false,
     "start_time": "2020-11-26T03:05:40.669720",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([0, 1])"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train['cp_dose'] = x_train['cp_dose'].map({\"D1\":0,\"D2\":1})\n",
    "x_train.cp_dose.unique()\n",
    "x_test['cp_dose'] = x_test['cp_dose'].map({\"D1\":0,\"D2\":1})\n",
    "x_test.cp_dose.unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:40.872641Z",
     "iopub.status.busy": "2020-11-26T03:05:40.871542Z",
     "iopub.status.idle": "2020-11-26T03:05:40.911844Z",
     "shell.execute_reply": "2020-11-26T03:05:40.912448Z"
    },
    "papermill": {
     "duration": 0.103835,
     "end_time": "2020-11-26T03:05:40.912631",
     "exception": false,
     "start_time": "2020-11-26T03:05:40.808796",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cp_time</th>\n",
       "      <th>cp_dose</th>\n",
       "      <th>g-0</th>\n",
       "      <th>g-1</th>\n",
       "      <th>g-2</th>\n",
       "      <th>g-3</th>\n",
       "      <th>g-4</th>\n",
       "      <th>g-5</th>\n",
       "      <th>g-6</th>\n",
       "      <th>g-7</th>\n",
       "      <th>...</th>\n",
       "      <th>c-90</th>\n",
       "      <th>c-91</th>\n",
       "      <th>c-92</th>\n",
       "      <th>c-93</th>\n",
       "      <th>c-94</th>\n",
       "      <th>c-95</th>\n",
       "      <th>c-96</th>\n",
       "      <th>c-97</th>\n",
       "      <th>c-98</th>\n",
       "      <th>c-99</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.0620</td>\n",
       "      <td>0.5577</td>\n",
       "      <td>-0.2479</td>\n",
       "      <td>-0.6208</td>\n",
       "      <td>-0.1944</td>\n",
       "      <td>-1.0120</td>\n",
       "      <td>-1.0220</td>\n",
       "      <td>-0.0326</td>\n",
       "      <td>...</td>\n",
       "      <td>0.2862</td>\n",
       "      <td>0.2584</td>\n",
       "      <td>0.8076</td>\n",
       "      <td>0.5523</td>\n",
       "      <td>-0.1912</td>\n",
       "      <td>0.6584</td>\n",
       "      <td>-0.3981</td>\n",
       "      <td>0.2139</td>\n",
       "      <td>0.3801</td>\n",
       "      <td>0.4176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0743</td>\n",
       "      <td>0.4087</td>\n",
       "      <td>0.2991</td>\n",
       "      <td>0.0604</td>\n",
       "      <td>1.0190</td>\n",
       "      <td>0.5207</td>\n",
       "      <td>0.2341</td>\n",
       "      <td>0.3372</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.4265</td>\n",
       "      <td>0.7543</td>\n",
       "      <td>0.4708</td>\n",
       "      <td>0.0230</td>\n",
       "      <td>0.2957</td>\n",
       "      <td>0.4899</td>\n",
       "      <td>0.1522</td>\n",
       "      <td>0.1241</td>\n",
       "      <td>0.6077</td>\n",
       "      <td>0.7371</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.6280</td>\n",
       "      <td>0.5817</td>\n",
       "      <td>1.5540</td>\n",
       "      <td>-0.0764</td>\n",
       "      <td>-0.0323</td>\n",
       "      <td>1.2390</td>\n",
       "      <td>0.1715</td>\n",
       "      <td>0.2155</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.7250</td>\n",
       "      <td>-0.6297</td>\n",
       "      <td>0.6103</td>\n",
       "      <td>0.0223</td>\n",
       "      <td>-1.3240</td>\n",
       "      <td>-0.3174</td>\n",
       "      <td>-0.6417</td>\n",
       "      <td>-0.2187</td>\n",
       "      <td>-1.4080</td>\n",
       "      <td>0.6931</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.5138</td>\n",
       "      <td>-0.2491</td>\n",
       "      <td>-0.2656</td>\n",
       "      <td>0.5288</td>\n",
       "      <td>4.0620</td>\n",
       "      <td>-0.8095</td>\n",
       "      <td>-1.9590</td>\n",
       "      <td>0.1792</td>\n",
       "      <td>...</td>\n",
       "      <td>-2.0990</td>\n",
       "      <td>-0.6441</td>\n",
       "      <td>-5.6300</td>\n",
       "      <td>-1.3780</td>\n",
       "      <td>-0.8632</td>\n",
       "      <td>-1.2880</td>\n",
       "      <td>-1.6210</td>\n",
       "      <td>-0.8784</td>\n",
       "      <td>-0.3876</td>\n",
       "      <td>-0.8154</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>-0.3254</td>\n",
       "      <td>-0.4009</td>\n",
       "      <td>0.9700</td>\n",
       "      <td>0.6919</td>\n",
       "      <td>1.4180</td>\n",
       "      <td>-0.8244</td>\n",
       "      <td>-0.2800</td>\n",
       "      <td>-0.1498</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0042</td>\n",
       "      <td>0.0048</td>\n",
       "      <td>0.6670</td>\n",
       "      <td>1.0690</td>\n",
       "      <td>0.5523</td>\n",
       "      <td>-0.3031</td>\n",
       "      <td>0.1094</td>\n",
       "      <td>0.2885</td>\n",
       "      <td>-0.3786</td>\n",
       "      <td>0.7125</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23809</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0.1394</td>\n",
       "      <td>-0.0636</td>\n",
       "      <td>-0.1112</td>\n",
       "      <td>-0.5080</td>\n",
       "      <td>-0.4713</td>\n",
       "      <td>0.7201</td>\n",
       "      <td>0.5773</td>\n",
       "      <td>0.3055</td>\n",
       "      <td>...</td>\n",
       "      <td>0.1969</td>\n",
       "      <td>0.0262</td>\n",
       "      <td>-0.8121</td>\n",
       "      <td>0.3434</td>\n",
       "      <td>0.5372</td>\n",
       "      <td>-0.3246</td>\n",
       "      <td>0.0631</td>\n",
       "      <td>0.9171</td>\n",
       "      <td>0.5258</td>\n",
       "      <td>0.4680</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23810</th>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>-1.3260</td>\n",
       "      <td>0.3478</td>\n",
       "      <td>-0.3743</td>\n",
       "      <td>0.9905</td>\n",
       "      <td>-0.7178</td>\n",
       "      <td>0.6621</td>\n",
       "      <td>-0.2252</td>\n",
       "      <td>-0.5565</td>\n",
       "      <td>...</td>\n",
       "      <td>0.4286</td>\n",
       "      <td>0.4426</td>\n",
       "      <td>0.0423</td>\n",
       "      <td>-0.3195</td>\n",
       "      <td>-0.8086</td>\n",
       "      <td>-0.9798</td>\n",
       "      <td>-0.2084</td>\n",
       "      <td>-0.1224</td>\n",
       "      <td>-0.2715</td>\n",
       "      <td>0.3689</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23811</th>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0.3942</td>\n",
       "      <td>0.3756</td>\n",
       "      <td>0.3109</td>\n",
       "      <td>-0.7389</td>\n",
       "      <td>0.5505</td>\n",
       "      <td>-0.0159</td>\n",
       "      <td>-0.2541</td>\n",
       "      <td>0.1745</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5409</td>\n",
       "      <td>0.3755</td>\n",
       "      <td>0.7343</td>\n",
       "      <td>0.2807</td>\n",
       "      <td>0.4116</td>\n",
       "      <td>0.6422</td>\n",
       "      <td>0.2256</td>\n",
       "      <td>0.7592</td>\n",
       "      <td>0.6656</td>\n",
       "      <td>0.3808</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23812</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.6660</td>\n",
       "      <td>0.2324</td>\n",
       "      <td>0.4392</td>\n",
       "      <td>0.2044</td>\n",
       "      <td>0.8531</td>\n",
       "      <td>-0.0343</td>\n",
       "      <td>0.0323</td>\n",
       "      <td>0.0463</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.1105</td>\n",
       "      <td>0.4258</td>\n",
       "      <td>-0.2012</td>\n",
       "      <td>0.1506</td>\n",
       "      <td>1.5230</td>\n",
       "      <td>0.7101</td>\n",
       "      <td>0.1732</td>\n",
       "      <td>0.7015</td>\n",
       "      <td>-0.6290</td>\n",
       "      <td>0.0740</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>23813</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.8598</td>\n",
       "      <td>1.0240</td>\n",
       "      <td>-0.1361</td>\n",
       "      <td>0.7952</td>\n",
       "      <td>-0.3611</td>\n",
       "      <td>-3.6750</td>\n",
       "      <td>-1.2420</td>\n",
       "      <td>0.9146</td>\n",
       "      <td>...</td>\n",
       "      <td>-3.3890</td>\n",
       "      <td>-1.7450</td>\n",
       "      <td>-6.6300</td>\n",
       "      <td>-4.0950</td>\n",
       "      <td>-7.3860</td>\n",
       "      <td>-1.4160</td>\n",
       "      <td>-3.5770</td>\n",
       "      <td>-0.4775</td>\n",
       "      <td>-2.1500</td>\n",
       "      <td>-4.2520</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>23814 rows × 874 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       cp_time  cp_dose     g-0     g-1     g-2     g-3     g-4     g-5  \\\n",
       "0            0        0  1.0620  0.5577 -0.2479 -0.6208 -0.1944 -1.0120   \n",
       "1            2        0  0.0743  0.4087  0.2991  0.0604  1.0190  0.5207   \n",
       "2            1        0  0.6280  0.5817  1.5540 -0.0764 -0.0323  1.2390   \n",
       "3            1        0 -0.5138 -0.2491 -0.2656  0.5288  4.0620 -0.8095   \n",
       "4            2        1 -0.3254 -0.4009  0.9700  0.6919  1.4180 -0.8244   \n",
       "...        ...      ...     ...     ...     ...     ...     ...     ...   \n",
       "23809        0        1  0.1394 -0.0636 -0.1112 -0.5080 -0.4713  0.7201   \n",
       "23810        0        1 -1.3260  0.3478 -0.3743  0.9905 -0.7178  0.6621   \n",
       "23811        1        1  0.3942  0.3756  0.3109 -0.7389  0.5505 -0.0159   \n",
       "23812        0        0  0.6660  0.2324  0.4392  0.2044  0.8531 -0.0343   \n",
       "23813        2        0 -0.8598  1.0240 -0.1361  0.7952 -0.3611 -3.6750   \n",
       "\n",
       "          g-6     g-7  ...    c-90    c-91    c-92    c-93    c-94    c-95  \\\n",
       "0     -1.0220 -0.0326  ...  0.2862  0.2584  0.8076  0.5523 -0.1912  0.6584   \n",
       "1      0.2341  0.3372  ... -0.4265  0.7543  0.4708  0.0230  0.2957  0.4899   \n",
       "2      0.1715  0.2155  ... -0.7250 -0.6297  0.6103  0.0223 -1.3240 -0.3174   \n",
       "3     -1.9590  0.1792  ... -2.0990 -0.6441 -5.6300 -1.3780 -0.8632 -1.2880   \n",
       "4     -0.2800 -0.1498  ...  0.0042  0.0048  0.6670  1.0690  0.5523 -0.3031   \n",
       "...       ...     ...  ...     ...     ...     ...     ...     ...     ...   \n",
       "23809  0.5773  0.3055  ...  0.1969  0.0262 -0.8121  0.3434  0.5372 -0.3246   \n",
       "23810 -0.2252 -0.5565  ...  0.4286  0.4426  0.0423 -0.3195 -0.8086 -0.9798   \n",
       "23811 -0.2541  0.1745  ...  0.5409  0.3755  0.7343  0.2807  0.4116  0.6422   \n",
       "23812  0.0323  0.0463  ... -0.1105  0.4258 -0.2012  0.1506  1.5230  0.7101   \n",
       "23813 -1.2420  0.9146  ... -3.3890 -1.7450 -6.6300 -4.0950 -7.3860 -1.4160   \n",
       "\n",
       "         c-96    c-97    c-98    c-99  \n",
       "0     -0.3981  0.2139  0.3801  0.4176  \n",
       "1      0.1522  0.1241  0.6077  0.7371  \n",
       "2     -0.6417 -0.2187 -1.4080  0.6931  \n",
       "3     -1.6210 -0.8784 -0.3876 -0.8154  \n",
       "4      0.1094  0.2885 -0.3786  0.7125  \n",
       "...       ...     ...     ...     ...  \n",
       "23809  0.0631  0.9171  0.5258  0.4680  \n",
       "23810 -0.2084 -0.1224 -0.2715  0.3689  \n",
       "23811  0.2256  0.7592  0.6656  0.3808  \n",
       "23812  0.1732  0.7015 -0.6290  0.0740  \n",
       "23813 -3.5770 -0.4775 -2.1500 -4.2520  \n",
       "\n",
       "[23814 rows x 874 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_train"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:41.073238Z",
     "iopub.status.busy": "2020-11-26T03:05:41.072292Z",
     "iopub.status.idle": "2020-11-26T03:05:41.076370Z",
     "shell.execute_reply": "2020-11-26T03:05:41.077120Z"
    },
    "papermill": {
     "duration": 0.094883,
     "end_time": "2020-11-26T03:05:41.077315",
     "exception": false,
     "start_time": "2020-11-26T03:05:40.982432",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "genes=[col for col in x_train.columns if col.startswith(\"g-\")]\n",
    "cells=[col for col in x_train.columns if col.startswith(\"c-\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:41.214595Z",
     "iopub.status.busy": "2020-11-26T03:05:41.213084Z",
     "iopub.status.idle": "2020-11-26T03:05:41.290469Z",
     "shell.execute_reply": "2020-11-26T03:05:41.289810Z"
    },
    "papermill": {
     "duration": 0.139019,
     "end_time": "2020-11-26T03:05:41.290611",
     "exception": false,
     "start_time": "2020-11-26T03:05:41.151592",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "a_train=x_train.copy()\n",
    "a_test=x_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:41.409302Z",
     "iopub.status.busy": "2020-11-26T03:05:41.407976Z",
     "iopub.status.idle": "2020-11-26T03:05:41.483703Z",
     "shell.execute_reply": "2020-11-26T03:05:41.483084Z"
    },
    "papermill": {
     "duration": 0.136956,
     "end_time": "2020-11-26T03:05:41.483838",
     "exception": false,
     "start_time": "2020-11-26T03:05:41.346882",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "x_train=a_train.copy()\n",
    "x_test=a_test.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:41.607725Z",
     "iopub.status.busy": "2020-11-26T03:05:41.606134Z",
     "iopub.status.idle": "2020-11-26T03:05:51.347702Z",
     "shell.execute_reply": "2020-11-26T03:05:51.346790Z"
    },
    "papermill": {
     "duration": 9.808232,
     "end_time": "2020-11-26T03:05:51.347835",
     "exception": false,
     "start_time": "2020-11-26T03:05:41.539603",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "scaler = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution=\"normal\")\n",
    "scaled_data=scaler.fit_transform(x_train[genes+cells])\n",
    "x_train=pd.concat([x_train.iloc[:,:2],pd.DataFrame(scaled_data,columns=genes+cells)],1)\n",
    "# scaled_data=scaler.transform(x_test[genes+cells])\n",
    "# x_test=pd.concat([x_test.iloc[:,:2],pd.DataFrame(scaled_data,columns=genes+cells)],1)\n",
    "\n",
    "\n",
    "transformer = QuantileTransformer(n_quantiles=100,random_state=42, output_distribution=\"normal\")\n",
    "scaled_data=scaler.fit_transform(x_test[genes+cells])\n",
    "x_test=pd.concat([x_test.iloc[:,:2],pd.DataFrame(scaled_data,columns=genes+cells)],1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:51.476884Z",
     "iopub.status.busy": "2020-11-26T03:05:51.475739Z",
     "iopub.status.idle": "2020-11-26T03:05:51.503609Z",
     "shell.execute_reply": "2020-11-26T03:05:51.504177Z"
    },
    "papermill": {
     "duration": 0.100162,
     "end_time": "2020-11-26T03:05:51.504324",
     "exception": false,
     "start_time": "2020-11-26T03:05:51.404162",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "-8977.505198164718"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "x_test.iloc[:,2:].sum().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:51.622508Z",
     "iopub.status.busy": "2020-11-26T03:05:51.621487Z",
     "iopub.status.idle": "2020-11-26T03:05:51.624682Z",
     "shell.execute_reply": "2020-11-26T03:05:51.624145Z"
    },
    "papermill": {
     "duration": 0.063781,
     "end_time": "2020-11-26T03:05:51.624811",
     "exception": false,
     "start_time": "2020-11-26T03:05:51.561030",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# genes = [cols for cols in x_train.columns if cols.startswith('c-')]\n",
    "# data=pd.concat([pd.DataFrame(x_train[genes]),pd.DataFrame(x_test[genes])])\n",
    "# g_data=PCA(0.95,random_state=42).fit_transform(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:51.742732Z",
     "iopub.status.busy": "2020-11-26T03:05:51.741699Z",
     "iopub.status.idle": "2020-11-26T03:05:51.744991Z",
     "shell.execute_reply": "2020-11-26T03:05:51.744425Z"
    },
    "papermill": {
     "duration": 0.063878,
     "end_time": "2020-11-26T03:05:51.745126",
     "exception": false,
     "start_time": "2020-11-26T03:05:51.681248",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# g_data.sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:51.924297Z",
     "iopub.status.busy": "2020-11-26T03:05:51.877231Z",
     "iopub.status.idle": "2020-11-26T03:05:57.013290Z",
     "shell.execute_reply": "2020-11-26T03:05:57.012390Z"
    },
    "papermill": {
     "duration": 5.211938,
     "end_time": "2020-11-26T03:05:57.013439",
     "exception": false,
     "start_time": "2020-11-26T03:05:51.801501",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "data=pd.concat((x_train[genes],x_test[genes]))\n",
    "g_data=PCA(n_components=0.95,random_state=44).fit_transform(data)\n",
    "\n",
    "g_train=g_data[:x_train.shape[0]]\n",
    "g_test=g_data[x_train.shape[0]:]\n",
    "\n",
    "g_train_df=pd.DataFrame(g_train,columns=[f'pca_g-{i}' for i in range(g_data.shape[1])])\n",
    "g_test_df=pd.DataFrame(g_test,columns=[f'pca_g-{i}' for i in range(g_data.shape[1])])\n",
    "\n",
    "\n",
    "data=pd.concat((x_train[cells],x_test[cells]))\n",
    "c_data=PCA(n_components=0.95,random_state=44).fit_transform(data)\n",
    "\n",
    "c_train=c_data[:x_train.shape[0]]\n",
    "c_test=c_data[x_train.shape[0]:]\n",
    "\n",
    "c_train_df=pd.DataFrame(c_train,columns=[f'pca_c-{i}' for i in range(c_data.shape[1])])\n",
    "c_test_df=pd.DataFrame(c_test,columns=[f'pca_c-{i}' for i in range(c_data.shape[1])])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:57.136664Z",
     "iopub.status.busy": "2020-11-26T03:05:57.135174Z",
     "iopub.status.idle": "2020-11-26T03:05:57.254347Z",
     "shell.execute_reply": "2020-11-26T03:05:57.253738Z"
    },
    "papermill": {
     "duration": 0.184327,
     "end_time": "2020-11-26T03:05:57.254476",
     "exception": false,
     "start_time": "2020-11-26T03:05:57.070149",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train=pd.concat((x_train,g_train_df,c_train_df),axis=1)\n",
    "X_test=pd.concat((x_test,g_test_df,c_test_df),axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:57.373735Z",
     "iopub.status.busy": "2020-11-26T03:05:57.372903Z",
     "iopub.status.idle": "2020-11-26T03:05:58.194773Z",
     "shell.execute_reply": "2020-11-26T03:05:58.194132Z"
    },
    "papermill": {
     "duration": 0.883996,
     "end_time": "2020-11-26T03:05:58.194909",
     "exception": false,
     "start_time": "2020-11-26T03:05:57.310913",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.feature_selection import VarianceThreshold\n",
    "varthre = VarianceThreshold(0.8)\n",
    "train_varthre=varthre.fit_transform(X_train.iloc[:,2:])\n",
    "test_varthre=varthre.transform(X_test.iloc[:,2:])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:58.318824Z",
     "iopub.status.busy": "2020-11-26T03:05:58.317319Z",
     "iopub.status.idle": "2020-11-26T03:05:58.428186Z",
     "shell.execute_reply": "2020-11-26T03:05:58.427183Z"
    },
    "papermill": {
     "duration": 0.177218,
     "end_time": "2020-11-26T03:05:58.428318",
     "exception": false,
     "start_time": "2020-11-26T03:05:58.251100",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "((23814, 1038), (3982, 1038), (23814, 206), (3982, 207))"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train=pd.concat([X_train.iloc[:,:2],pd.DataFrame(train_varthre,columns=X_train.columns[2:][varthre.get_support()])],1)\n",
    "X_test=pd.concat([X_test.iloc[:,:2],pd.DataFrame(test_varthre,columns=X_test.columns[2:][varthre.get_support()])],1)\n",
    "\n",
    "X_train.shape,X_test.shape,y_train.shape,submission.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:58.571089Z",
     "iopub.status.busy": "2020-11-26T03:05:58.566509Z",
     "iopub.status.idle": "2020-11-26T03:05:58.607050Z",
     "shell.execute_reply": "2020-11-26T03:05:58.606305Z"
    },
    "papermill": {
     "duration": 0.122044,
     "end_time": "2020-11-26T03:05:58.607195",
     "exception": false,
     "start_time": "2020-11-26T03:05:58.485151",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cp_time</th>\n",
       "      <th>cp_dose</th>\n",
       "      <th>g-0</th>\n",
       "      <th>g-1</th>\n",
       "      <th>g-2</th>\n",
       "      <th>g-3</th>\n",
       "      <th>g-4</th>\n",
       "      <th>g-5</th>\n",
       "      <th>g-6</th>\n",
       "      <th>g-7</th>\n",
       "      <th>...</th>\n",
       "      <th>pca_g-154</th>\n",
       "      <th>pca_g-155</th>\n",
       "      <th>pca_c-0</th>\n",
       "      <th>pca_c-1</th>\n",
       "      <th>pca_c-2</th>\n",
       "      <th>pca_c-3</th>\n",
       "      <th>pca_c-4</th>\n",
       "      <th>pca_c-5</th>\n",
       "      <th>pca_c-6</th>\n",
       "      <th>pca_c-7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1.134849</td>\n",
       "      <td>0.907687</td>\n",
       "      <td>-0.416385</td>\n",
       "      <td>-0.966814</td>\n",
       "      <td>-0.254723</td>\n",
       "      <td>-1.017473</td>\n",
       "      <td>-1.364787</td>\n",
       "      <td>-0.024007</td>\n",
       "      <td>...</td>\n",
       "      <td>0.003567</td>\n",
       "      <td>1.595561</td>\n",
       "      <td>4.917651</td>\n",
       "      <td>1.538645</td>\n",
       "      <td>-1.560099</td>\n",
       "      <td>1.127068</td>\n",
       "      <td>0.908941</td>\n",
       "      <td>-1.080358</td>\n",
       "      <td>-0.253258</td>\n",
       "      <td>0.115213</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0.119282</td>\n",
       "      <td>0.681738</td>\n",
       "      <td>0.272399</td>\n",
       "      <td>0.080113</td>\n",
       "      <td>1.205169</td>\n",
       "      <td>0.686517</td>\n",
       "      <td>0.313396</td>\n",
       "      <td>0.555877</td>\n",
       "      <td>...</td>\n",
       "      <td>-1.268543</td>\n",
       "      <td>-0.743488</td>\n",
       "      <td>5.090695</td>\n",
       "      <td>-0.368828</td>\n",
       "      <td>0.013911</td>\n",
       "      <td>0.994664</td>\n",
       "      <td>-0.502353</td>\n",
       "      <td>0.673057</td>\n",
       "      <td>-0.272862</td>\n",
       "      <td>0.455034</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0.779973</td>\n",
       "      <td>0.946463</td>\n",
       "      <td>1.425350</td>\n",
       "      <td>-0.132928</td>\n",
       "      <td>-0.006122</td>\n",
       "      <td>1.492493</td>\n",
       "      <td>0.235577</td>\n",
       "      <td>0.365511</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.876965</td>\n",
       "      <td>-0.960548</td>\n",
       "      <td>-1.352416</td>\n",
       "      <td>0.294769</td>\n",
       "      <td>-0.291118</td>\n",
       "      <td>-0.218369</td>\n",
       "      <td>-0.098231</td>\n",
       "      <td>-0.711736</td>\n",
       "      <td>0.893185</td>\n",
       "      <td>0.039182</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 1038 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cp_time  cp_dose       g-0       g-1       g-2       g-3       g-4  \\\n",
       "0        0        0  1.134849  0.907687 -0.416385 -0.966814 -0.254723   \n",
       "1        2        0  0.119282  0.681738  0.272399  0.080113  1.205169   \n",
       "2        1        0  0.779973  0.946463  1.425350 -0.132928 -0.006122   \n",
       "\n",
       "        g-5       g-6       g-7  ...  pca_g-154  pca_g-155   pca_c-0  \\\n",
       "0 -1.017473 -1.364787 -0.024007  ...   0.003567   1.595561  4.917651   \n",
       "1  0.686517  0.313396  0.555877  ...  -1.268543  -0.743488  5.090695   \n",
       "2  1.492493  0.235577  0.365511  ...  -0.876965  -0.960548 -1.352416   \n",
       "\n",
       "    pca_c-1   pca_c-2   pca_c-3   pca_c-4   pca_c-5   pca_c-6   pca_c-7  \n",
       "0  1.538645 -1.560099  1.127068  0.908941 -1.080358 -0.253258  0.115213  \n",
       "1 -0.368828  0.013911  0.994664 -0.502353  0.673057 -0.272862  0.455034  \n",
       "2  0.294769 -0.291118 -0.218369 -0.098231 -0.711736  0.893185  0.039182  \n",
       "\n",
       "[3 rows x 1038 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(23814, 1038)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>cp_time</th>\n",
       "      <th>cp_dose</th>\n",
       "      <th>g-0</th>\n",
       "      <th>g-1</th>\n",
       "      <th>g-2</th>\n",
       "      <th>g-3</th>\n",
       "      <th>g-4</th>\n",
       "      <th>g-5</th>\n",
       "      <th>g-6</th>\n",
       "      <th>g-7</th>\n",
       "      <th>...</th>\n",
       "      <th>pca_g-154</th>\n",
       "      <th>pca_g-155</th>\n",
       "      <th>pca_c-0</th>\n",
       "      <th>pca_c-1</th>\n",
       "      <th>pca_c-2</th>\n",
       "      <th>pca_c-3</th>\n",
       "      <th>pca_c-4</th>\n",
       "      <th>pca_c-5</th>\n",
       "      <th>pca_c-6</th>\n",
       "      <th>pca_c-7</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.755294</td>\n",
       "      <td>0.214796</td>\n",
       "      <td>-0.774511</td>\n",
       "      <td>0.705349</td>\n",
       "      <td>1.564580</td>\n",
       "      <td>-0.194968</td>\n",
       "      <td>-0.276944</td>\n",
       "      <td>0.363393</td>\n",
       "      <td>...</td>\n",
       "      <td>0.494774</td>\n",
       "      <td>0.372851</td>\n",
       "      <td>-0.804259</td>\n",
       "      <td>-1.899650</td>\n",
       "      <td>0.657893</td>\n",
       "      <td>-0.338125</td>\n",
       "      <td>0.099395</td>\n",
       "      <td>-0.800232</td>\n",
       "      <td>0.523428</td>\n",
       "      <td>-0.393640</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>-0.186773</td>\n",
       "      <td>0.374643</td>\n",
       "      <td>1.176402</td>\n",
       "      <td>-0.652299</td>\n",
       "      <td>-0.546638</td>\n",
       "      <td>-0.403447</td>\n",
       "      <td>-2.325903</td>\n",
       "      <td>0.645634</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.610010</td>\n",
       "      <td>0.790070</td>\n",
       "      <td>-5.397308</td>\n",
       "      <td>-0.456508</td>\n",
       "      <td>0.290936</td>\n",
       "      <td>-1.095456</td>\n",
       "      <td>-0.612847</td>\n",
       "      <td>1.042001</td>\n",
       "      <td>0.832567</td>\n",
       "      <td>1.149697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.351229</td>\n",
       "      <td>-0.155308</td>\n",
       "      <td>-0.613250</td>\n",
       "      <td>0.255053</td>\n",
       "      <td>-1.762152</td>\n",
       "      <td>0.333321</td>\n",
       "      <td>-0.434423</td>\n",
       "      <td>-0.484560</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.026030</td>\n",
       "      <td>-0.539568</td>\n",
       "      <td>3.589625</td>\n",
       "      <td>1.289070</td>\n",
       "      <td>0.156473</td>\n",
       "      <td>-0.164602</td>\n",
       "      <td>-1.030081</td>\n",
       "      <td>2.044905</td>\n",
       "      <td>-0.786719</td>\n",
       "      <td>-0.974288</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3 rows × 1038 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   cp_time  cp_dose       g-0       g-1       g-2       g-3       g-4  \\\n",
       "0        0        0 -0.755294  0.214796 -0.774511  0.705349  1.564580   \n",
       "1        2        0 -0.186773  0.374643  1.176402 -0.652299 -0.546638   \n",
       "2        0        0  0.351229 -0.155308 -0.613250  0.255053 -1.762152   \n",
       "\n",
       "        g-5       g-6       g-7  ...  pca_g-154  pca_g-155   pca_c-0  \\\n",
       "0 -0.194968 -0.276944  0.363393  ...   0.494774   0.372851 -0.804259   \n",
       "1 -0.403447 -2.325903  0.645634  ...  -0.610010   0.790070 -5.397308   \n",
       "2  0.333321 -0.434423 -0.484560  ...  -0.026030  -0.539568  3.589625   \n",
       "\n",
       "    pca_c-1   pca_c-2   pca_c-3   pca_c-4   pca_c-5   pca_c-6   pca_c-7  \n",
       "0 -1.899650  0.657893 -0.338125  0.099395 -0.800232  0.523428 -0.393640  \n",
       "1 -0.456508  0.290936 -1.095456 -0.612847  1.042001  0.832567  1.149697  \n",
       "2  1.289070  0.156473 -0.164602 -1.030081  2.044905 -0.786719 -0.974288  \n",
       "\n",
       "[3 rows x 1038 columns]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3982, 1038)\n"
     ]
    }
   ],
   "source": [
    "display(X_train.head(3))\n",
    "print(X_train.shape)\n",
    "display(X_test.head(3))\n",
    "print(X_test.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:58.738365Z",
     "iopub.status.busy": "2020-11-26T03:05:58.737633Z",
     "iopub.status.idle": "2020-11-26T03:05:58.742521Z",
     "shell.execute_reply": "2020-11-26T03:05:58.741931Z"
    },
    "papermill": {
     "duration": 0.069933,
     "end_time": "2020-11-26T03:05:58.742658",
     "exception": false,
     "start_time": "2020-11-26T03:05:58.672725",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "genes=[col for col in X_train.columns if col.startswith(\"g-\")]\n",
    "cells=[col for col in X_train.columns if col.startswith(\"c-\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:05:58.874814Z",
     "iopub.status.busy": "2020-11-26T03:05:58.873549Z",
     "iopub.status.idle": "2020-11-26T03:07:42.871656Z",
     "shell.execute_reply": "2020-11-26T03:07:42.870601Z"
    },
    "papermill": {
     "duration": 104.069916,
     "end_time": "2020-11-26T03:07:42.871799",
     "exception": false,
     "start_time": "2020-11-26T03:05:58.801883",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.cluster import KMeans\n",
    "# kmeans=KMeans(n_clusters=40,random_state=982)\n",
    "kmeans=KMeans(n_clusters=35,random_state=982)\n",
    "kmeans.fit(pd.concat([X_train[genes],X_test[genes]]))\n",
    "X_train[\"cluster_g\"]=kmeans.labels_[:len(X_train)]\n",
    "X_test[\"cluster_g\"]=kmeans.labels_[len(X_train):]\n",
    "\n",
    "from sklearn.cluster import KMeans\n",
    "# kmeans=KMeans(n_clusters=8,random_state=982)\n",
    "kmeans=KMeans(n_clusters=5,random_state=982)\n",
    "kmeans.fit(pd.concat([X_train[cells],X_test[cells]]))\n",
    "X_train[\"cluster_c\"]=kmeans.labels_[:len(X_train)]\n",
    "X_test[\"cluster_c\"]=kmeans.labels_[len(X_train):]\n",
    "\n",
    "X_train=pd.get_dummies(X_train,columns=['cluster_g','cluster_c'])\n",
    "X_test=pd.get_dummies(X_test,columns=['cluster_g','cluster_c'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:43.009929Z",
     "iopub.status.busy": "2020-11-26T03:07:43.008927Z",
     "iopub.status.idle": "2020-11-26T03:07:43.012935Z",
     "shell.execute_reply": "2020-11-26T03:07:43.011915Z"
    },
    "papermill": {
     "duration": 0.081835,
     "end_time": "2020-11-26T03:07:43.013063",
     "exception": false,
     "start_time": "2020-11-26T03:07:42.931228",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "gsquarecols=['g-574','g-211','g-216','g-0','g-255','g-577','g-153','g-389','g-60','g-370','g-248','g-167','g-203','g-177','g-301','g-332','g-517','g-6','g-744','g-224','g-162','g-3','g-736','g-486','g-283','g-22','g-359','g-361','g-440','g-335','g-106','g-307','g-745','g-146','g-416','g-298','g-666','g-91','g-17','g-549','g-145','g-157','g-768','g-568','g-396']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:43.155072Z",
     "iopub.status.busy": "2020-11-26T03:07:43.150731Z",
     "iopub.status.idle": "2020-11-26T03:07:56.619285Z",
     "shell.execute_reply": "2020-11-26T03:07:56.620005Z"
    },
    "papermill": {
     "duration": 13.545386,
     "end_time": "2020-11-26T03:07:56.620173",
     "exception": false,
     "start_time": "2020-11-26T03:07:43.074787",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for df in X_train,X_test:\n",
    "    df['time_dose']=df['cp_time']+df['cp_dose']\n",
    "    df['g_median']=df[genes].median(axis=1)\n",
    "    df['c_median']=df[cells].median(axis=1)\n",
    "    df['gc_median']=df[genes+cells].median(axis=1)\n",
    "    df['g_max']=df[genes].max(axis=1)\n",
    "    df['c_max']=df[cells].max(axis=1)\n",
    "    df['gc_max']=df[genes+cells].max(axis=1)\n",
    "    df['g_min']=df[genes].min(axis=1)\n",
    "    df['c_min']=df[cells].min(axis=1)\n",
    "    df['gc_min']=df[genes+cells].min(axis=1)\n",
    "    df['g_sum']=df[genes].sum(axis=1)\n",
    "    df['c_sum']=df[cells].sum(axis=1)\n",
    "    df['gc_sum']=df[genes+cells].sum(axis=1)\n",
    "    df['g_mean']=df[genes].mean(axis=1)\n",
    "    df['c_mean']=df[cells].mean(axis=1)\n",
    "    df['gc_mean']=df[genes+cells].mean(axis=1)\n",
    "    df['g_std']=df[genes].std(axis=1)\n",
    "    df['c_std']=df[cells].std(axis=1)\n",
    "    df['gc_std']=df[genes+cells].std(axis=1)\n",
    "    df['g_skew']=df[genes].skew(axis=1)\n",
    "    df['c_skew']=df[cells].skew(axis=1)\n",
    "    df['gc_skew']=df[genes+cells].skew(axis=1)\n",
    "    df['g_kurtosis']=df[genes].kurtosis(axis=1)\n",
    "    df['c_kurtosis']=df[cells].kurtosis(axis=1)\n",
    "    df['gc_kurtosis']=df[genes+cells].kurtosis(axis=1)\n",
    "    df['g_q25']=df[genes].quantile(0.25,axis=1)\n",
    "    df['c_q25']=df[cells].quantile(0.25,axis=1)\n",
    "    df['gc_q25']=df[genes+cells].quantile(0.25,axis=1)\n",
    "    df['g_q75']=df[genes].quantile(0.75,axis=1)\n",
    "    df['c_q75']=df[cells].quantile(0.75,axis=1)\n",
    "    df['gc_q75']=df[genes+cells].quantile(0.75,axis=1)\n",
    "    \n",
    "    \n",
    "    df['c52_c42'] = df['c-52'] * df['c-42']\n",
    "    df['c13_c73'] = df['c-13'] * df['c-73']\n",
    "    df['c26_c13'] = df['c-23'] * df['c-13']\n",
    "    df['c33_c6'] = df['c-33'] * df['c-6']\n",
    "    df['c11_c55'] = df['c-11'] * df['c-55']\n",
    "    df['c38_c63'] = df['c-38'] * df['c-63']\n",
    "    df['c38_c94'] = df['c-38'] * df['c-94']\n",
    "    df['c13_c94'] = df['c-13'] * df['c-94']\n",
    "    df['c4_c52'] = df['c-4'] * df['c-52']\n",
    "    df['c4_c42'] = df['c-4'] * df['c-42']\n",
    "    df['c13_c38'] = df['c-13'] * df['c-38']\n",
    "    df['c55_c2'] = df['c-55'] * df['c-2']\n",
    "    df['c55_c4'] = df['c-55'] * df['c-4']\n",
    "    df['c4_c13'] = df['c-4'] * df['c-13']\n",
    "    df['c82_c42'] = df['c-82'] * df['c-42']\n",
    "    df['c66_c42'] = df['c-66'] * df['c-42']\n",
    "    df['c6_c38'] = df['c-6'] * df['c-38']\n",
    "    df['c2_c13'] = df['c-2'] * df['c-13']\n",
    "    df['c62_c42'] = df['c-62'] * df['c-42']\n",
    "    df['c90_c55'] = df['c-90'] * df['c-55']\n",
    "    \n",
    "    for feature in cells:\n",
    "             df[f'{feature}_squared'] = df[feature] ** 2     \n",
    "                \n",
    "    for feature in gsquarecols:\n",
    "        df[f'{feature}_squared'] = df[feature] ** 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:56.750162Z",
     "iopub.status.busy": "2020-11-26T03:07:56.749103Z",
     "iopub.status.idle": "2020-11-26T03:07:56.756334Z",
     "shell.execute_reply": "2020-11-26T03:07:56.757210Z"
    },
    "papermill": {
     "duration": 0.071264,
     "end_time": "2020-11-26T03:07:56.757404",
     "exception": false,
     "start_time": "2020-11-26T03:07:56.686140",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(23814, 1274)\n",
      "(3982, 1274)\n",
      "(23814, 206)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:56.889697Z",
     "iopub.status.busy": "2020-11-26T03:07:56.888639Z",
     "iopub.status.idle": "2020-11-26T03:07:57.167639Z",
     "shell.execute_reply": "2020-11-26T03:07:57.166936Z"
    },
    "papermill": {
     "duration": 0.344586,
     "end_time": "2020-11-26T03:07:57.167763",
     "exception": false,
     "start_time": "2020-11-26T03:07:56.823177",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "X_train=X_train.loc[trt_cp_index,:]\n",
    "y_train=y_train.loc[trt_cp_index,:]\n",
    "y_train_non=y_train_non.loc[trt_cp_index,:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:57.295808Z",
     "iopub.status.busy": "2020-11-26T03:07:57.294879Z",
     "iopub.status.idle": "2020-11-26T03:07:57.301293Z",
     "shell.execute_reply": "2020-11-26T03:07:57.300526Z"
    },
    "papermill": {
     "duration": 0.073264,
     "end_time": "2020-11-26T03:07:57.301415",
     "exception": false,
     "start_time": "2020-11-26T03:07:57.228151",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(21948, 1274)\n",
      "(3982, 1274)\n",
      "(21948, 206)\n",
      "(21948, 403)\n"
     ]
    }
   ],
   "source": [
    "print(X_train.shape)\n",
    "print(X_test.shape)\n",
    "print(y_train.shape)\n",
    "print(y_train_non.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:57.436745Z",
     "iopub.status.busy": "2020-11-26T03:07:57.430619Z",
     "iopub.status.idle": "2020-11-26T03:07:57.439534Z",
     "shell.execute_reply": "2020-11-26T03:07:57.438987Z"
    },
    "papermill": {
     "duration": 0.078184,
     "end_time": "2020-11-26T03:07:57.439667",
     "exception": false,
     "start_time": "2020-11-26T03:07:57.361483",
     "status": "completed"
    },
    "scrolled": false,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# def getmodel(layers,dropouts,activations1,activations2,layer_sizes1,layer_sizes2,lr,wd):\n",
    "#     model=tf.keras.Sequential()\n",
    "#     model.add(tf.keras.Input(shape=(689,)))\n",
    "    \n",
    "    \n",
    "#     for i in range(layers):\n",
    "#         model.add(\n",
    "#             tf.keras.layers.BatchNormalization()\n",
    "#         )\n",
    "#         model.add(\n",
    "#             tf.keras.layers.Dropout(dropouts[i])\n",
    "#         )\n",
    "#         model.add(\n",
    "#             tfa.layers.WeightNormalization(tf.keras.layers.Dense(layer_sizes1[i],activation=activations1[i]))\n",
    "#         )\n",
    "#         model.add(\n",
    "#             tf.keras.layers.Dense(layer_sizes2[i],activation=activations2[i])\n",
    "#         )\n",
    "        \n",
    "        \n",
    "#     model.add(tf.keras.layers.Dense(206,activation='softmax'))\n",
    "    \n",
    "# #     model.compile(optimizer=tfa.optimizers.AdamW(lr = 1e-3, weight_decay = 1e-5, clipvalue = 756),loss=tf.keras.losses.BinaryCrossentropy(),metrics=['accuracy'])\n",
    "#     model.compile(optimizer=tfa.optimizers.AdamW(lr = lr, weight_decay = wd, clipvalue = 756),loss=tf.keras.losses.BinaryCrossentropy(),metrics=['accuracy'])\n",
    "    \n",
    "#     return model\n",
    "\n",
    "\n",
    "# def get_model(layers,units,dropouts,activations):\n",
    "#     model=tf.keras.Sequential()\n",
    "#     model.add(tf.keras.Input(shape=(874,)))\n",
    "#     model.add(tf.keras.layers.BatchNormalization())\n",
    "        \n",
    "#     for i in range(layers):\n",
    "#         model.add(tf.keras.layers.Dense(units[i], activation=activations[i]))\n",
    "#         model.add(tf.keras.layers.Dropout(dropouts[i]))\n",
    "#         model.add(tf.keras.layers.BatchNormalization())\n",
    "            \n",
    "#     model.add(tf.keras.layers.Dense(206,activation='softmax'))\n",
    "    \n",
    "#     model.compile(optimizer = tf.keras.optimizers.Adam(1e-3),loss=tf.keras.losses.BinaryCrossentropy(), metrics=['accuracy'])\n",
    "#     return model\n",
    "\n",
    "def create_model(num_cols, hid_layers, activations, dropout_rate, lr, num_cols_y):\n",
    "    \n",
    "    inp1 = tf.keras.layers.Input(shape = (num_cols, ))\n",
    "    x1 = tf.keras.layers.BatchNormalization()(inp1)\n",
    "\n",
    "    for i, units in enumerate(hid_layers):\n",
    "#         x1 = tfa.layers.WeightNormalization(tf.keras.layers.Dense(units, activation=activations[i]))(x1)\n",
    "        x1 = tf.keras.layers.Dense(units, activation=activations[i])(x1)\n",
    "        x1 = tf.keras.layers.Dropout(dropout_rate[i])(x1)\n",
    "        x1 = tf.keras.layers.BatchNormalization()(x1)\n",
    "    \n",
    "    x1 = tf.keras.layers.Dense(num_cols_y,activation='sigmoid')(x1)\n",
    "    model = tf.keras.models.Model(inputs= inp1, outputs= x1)\n",
    "    \n",
    "    model.compile(optimizer = tf.keras.optimizers.Adam(learning_rate=lr),loss=tf.keras.losses.BinaryCrossentropy(label_smoothing=0.001), metrics=['accuracy'])\n",
    "#     model.compile(optimizer=tfa.optimizers.AdamW(lr = 1e-3, weight_decay = 1e-5, clipvalue = 756),loss=tf.keras.losses.BinaryCrossentropy(),metrics=['accuracy'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:57.590571Z",
     "iopub.status.busy": "2020-11-26T03:07:57.588249Z",
     "iopub.status.idle": "2020-11-26T03:07:57.591326Z",
     "shell.execute_reply": "2020-11-26T03:07:57.591892Z"
    },
    "papermill": {
     "duration": 0.091833,
     "end_time": "2020-11-26T03:07:57.592040",
     "exception": false,
     "start_time": "2020-11-26T03:07:57.500207",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "hps={\n",
    "    30:[\n",
    "    {\n",
    "        \"units\":[1410,1246,512,512,512],\n",
    "        \"activations\":['selu', 'swish', 'elu','elu','elu'],\n",
    "        \"dropouts\":[0.49,0.51,0.0,0.0,0.0]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[1127,1133, 1209, 512],\n",
    "        \"activations\":[ 'selu','swish','swish','elu'],\n",
    "        \"dropouts\":[0.32,0.02,0.26,0.0]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[796,1995,573],\n",
    "        \"activations\":['selu','selu','swish'],\n",
    "        \"dropouts\":[0.22,0.21,0.09]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[959,1557,1826],\n",
    "        \"activations\":['selu','selu','elu'],\n",
    "        \"dropouts\":[0.42,0.61,0.3]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[1081,2033],\n",
    "        \"activations\":['elu','elu'],\n",
    "        \"dropouts\":[0.44,0.12]\n",
    "    }    \n",
    "    ],\n",
    "    58:[\n",
    "        {\n",
    "            \"units\":[561,882,2023],\n",
    "            \"activations\":['elu','elu','elu'],\n",
    "            \"dropouts\":[0.43,0.17,0.29]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[687,1085,1365,1427,1325],\n",
    "            \"activations\":['selu','swish','swish','elu','selu'],\n",
    "            \"dropouts\":[0.59,0.15,0.31,0.08,0.41]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1324,1475,976,512],\n",
    "            \"activations\":['selu','selu','selu', 'elu'],\n",
    "            \"dropouts\":[0.39,0.38,0.19,0.0]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1410,1246,512,512,512],\n",
    "            \"activations\":['selu', 'swish', 'elu', 'elu', 'elu'],\n",
    "            \"dropouts\":[0.49,0.51,0.0,0.0,0.0]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[750,1734,638,512],\n",
    "            \"activations\":['selu','swish','selu','elu'],\n",
    "            \"dropouts\":[0.42,0.5,0.42,0.0]\n",
    "        }\n",
    "    ],\n",
    "    3255:[\n",
    "        {\n",
    "            \"units\":[1444,629,512,512,512],\n",
    "            \"activations\":['selu', 'swish', 'elu', 'elu', 'elu'],\n",
    "            \"dropouts\":[0.62,0.4,0.0,0.0,0.0]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[899,940,893,718],\n",
    "            \"activations\":['elu', 'elu', 'selu', 'selu'],\n",
    "            \"dropouts\":[0.33,0.28,0.29,0.17]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[2030,1422],\n",
    "            \"activations\":['elu', 'selu'],\n",
    "            \"dropouts\":[0.53,0.37]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1194,1562],\n",
    "            \"activations\":[ 'swish','swish'],\n",
    "            \"dropouts\":[0.59,0.0]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1130,766,520,512],\n",
    "            \"activations\":[  'selu','elu','elu', 'elu'],\n",
    "            \"dropouts\":[0.29,0.14,0.27,0.0]\n",
    "        }\n",
    "    ]\n",
    "}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:57.718012Z",
     "iopub.status.busy": "2020-11-26T03:07:57.717101Z",
     "iopub.status.idle": "2020-11-26T03:07:57.721375Z",
     "shell.execute_reply": "2020-11-26T03:07:57.721906Z"
    },
    "papermill": {
     "duration": 0.06995,
     "end_time": "2020-11-26T03:07:57.722061",
     "exception": false,
     "start_time": "2020-11-26T03:07:57.652111",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from pickle import load,dump"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:07:57.886693Z",
     "iopub.status.busy": "2020-11-26T03:07:57.862359Z",
     "iopub.status.idle": "2020-11-26T03:16:58.844253Z",
     "shell.execute_reply": "2020-11-26T03:16:58.843540Z"
    },
    "papermill": {
     "duration": 541.062096,
     "end_time": "2020-11-26T03:16:58.844394",
     "exception": false,
     "start_time": "2020-11-26T03:07:57.782298",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=30 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:30 Fold:0\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.2900 - accuracy: 0.0302 - val_loss: 0.0233 - val_accuracy: 0.0483\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0168 - accuracy: 0.0415 - val_loss: 0.0145 - val_accuracy: 0.0335\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0138 - accuracy: 0.0599 - val_loss: 0.0135 - val_accuracy: 0.0647\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0132 - accuracy: 0.0753 - val_loss: 0.0129 - val_accuracy: 0.0818\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0127 - accuracy: 0.1034 - val_loss: 0.0125 - val_accuracy: 0.1059\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0125 - accuracy: 0.1189 - val_loss: 0.0124 - val_accuracy: 0.1096\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0122 - accuracy: 0.1357 - val_loss: 0.0122 - val_accuracy: 0.1378\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0120 - accuracy: 0.1423 - val_loss: 0.0121 - val_accuracy: 0.1401\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0118 - accuracy: 0.1594 - val_loss: 0.0121 - val_accuracy: 0.1392\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0116 - accuracy: 0.1644 - val_loss: 0.0120 - val_accuracy: 0.1405\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0114 - accuracy: 0.1787 - val_loss: 0.0121 - val_accuracy: 0.1476\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0112 - accuracy: 0.1897 - val_loss: 0.0120 - val_accuracy: 0.1508\n",
      "Epoch 13/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0110 - accuracy: 0.2067\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0110 - accuracy: 0.2061 - val_loss: 0.0121 - val_accuracy: 0.1517\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0104 - accuracy: 0.2419 - val_loss: 0.0118 - val_accuracy: 0.1599\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0102 - accuracy: 0.2484 - val_loss: 0.0118 - val_accuracy: 0.1640\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0101 - accuracy: 0.2558 - val_loss: 0.0118 - val_accuracy: 0.1633\n",
      "Epoch 17/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0101 - accuracy: 0.2602\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0101 - accuracy: 0.2606 - val_loss: 0.0118 - val_accuracy: 0.1642\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0099 - accuracy: 0.2714 - val_loss: 0.0119 - val_accuracy: 0.1649\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0099 - accuracy: 0.2727 - val_loss: 0.0119 - val_accuracy: 0.1645\n",
      "Epoch 20/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0099 - accuracy: 0.2716\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0099 - accuracy: 0.2721 - val_loss: 0.0119 - val_accuracy: 0.1647\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0099 - accuracy: 0.2721 - val_loss: 0.0119 - val_accuracy: 0.1645\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0099 - accuracy: 0.2720 - val_loss: 0.0119 - val_accuracy: 0.1645\n",
      "Epoch 23/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0099 - accuracy: 0.2764\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0099 - accuracy: 0.2759 - val_loss: 0.0119 - val_accuracy: 0.1645\n",
      "Epoch 24/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0099 - accuracy: 0.2734Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0099 - accuracy: 0.2729 - val_loss: 0.0119 - val_accuracy: 0.1645\n",
      "Epoch 00024: early stopping\n",
      "SEED:30 Fold:1\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2909 - accuracy: 0.0350 - val_loss: 0.0232 - val_accuracy: 0.0410\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0167 - accuracy: 0.0496 - val_loss: 0.0145 - val_accuracy: 0.0781\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0137 - accuracy: 0.0750 - val_loss: 0.0134 - val_accuracy: 0.1116\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.1031 - val_loss: 0.0128 - val_accuracy: 0.1046\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0125 - accuracy: 0.1206 - val_loss: 0.0124 - val_accuracy: 0.1173\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0121 - accuracy: 0.1467 - val_loss: 0.0124 - val_accuracy: 0.1244\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0118 - accuracy: 0.1563 - val_loss: 0.0123 - val_accuracy: 0.1449\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0115 - accuracy: 0.1819 - val_loss: 0.0121 - val_accuracy: 0.1620\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0112 - accuracy: 0.2060 - val_loss: 0.0121 - val_accuracy: 0.1497\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0108 - accuracy: 0.2291 - val_loss: 0.0122 - val_accuracy: 0.1510\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0103 - accuracy: 0.2719\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0103 - accuracy: 0.2719 - val_loss: 0.0123 - val_accuracy: 0.1510\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0094 - accuracy: 0.3458 - val_loss: 0.0120 - val_accuracy: 0.1579\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0090 - accuracy: 0.3802 - val_loss: 0.0120 - val_accuracy: 0.1736\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.4070 - val_loss: 0.0121 - val_accuracy: 0.1626\n",
      "Epoch 15/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.4300\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0086 - accuracy: 0.4298 - val_loss: 0.0121 - val_accuracy: 0.1649\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0084 - accuracy: 0.4532 - val_loss: 0.0121 - val_accuracy: 0.1633\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0083 - accuracy: 0.4543 - val_loss: 0.0121 - val_accuracy: 0.1631\n",
      "Epoch 18/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0083 - accuracy: 0.4564\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0083 - accuracy: 0.4563 - val_loss: 0.0122 - val_accuracy: 0.1638\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0083 - accuracy: 0.4622 - val_loss: 0.0122 - val_accuracy: 0.1642\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0083 - accuracy: 0.4614 - val_loss: 0.0122 - val_accuracy: 0.1640\n",
      "Epoch 21/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0083 - accuracy: 0.4612\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0083 - accuracy: 0.4613 - val_loss: 0.0122 - val_accuracy: 0.1638\n",
      "Epoch 22/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0083 - accuracy: 0.4648Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0083 - accuracy: 0.4643 - val_loss: 0.0122 - val_accuracy: 0.1640\n",
      "Epoch 00022: early stopping\n",
      "SEED:30 Fold:2\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2959 - accuracy: 0.0267 - val_loss: 0.0209 - val_accuracy: 0.0310\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0174 - accuracy: 0.0408 - val_loss: 0.0146 - val_accuracy: 0.0583\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0142 - accuracy: 0.0576 - val_loss: 0.0135 - val_accuracy: 0.0535\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0136 - accuracy: 0.0739 - val_loss: 0.0132 - val_accuracy: 0.0720\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0131 - accuracy: 0.0980 - val_loss: 0.0129 - val_accuracy: 0.1180\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0128 - accuracy: 0.1139 - val_loss: 0.0127 - val_accuracy: 0.1030\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0126 - accuracy: 0.1256 - val_loss: 0.0127 - val_accuracy: 0.1066\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0124 - accuracy: 0.1283 - val_loss: 0.0125 - val_accuracy: 0.1476\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0122 - accuracy: 0.1445 - val_loss: 0.0124 - val_accuracy: 0.1269\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0120 - accuracy: 0.1478 - val_loss: 0.0124 - val_accuracy: 0.1308\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0118 - accuracy: 0.1641 - val_loss: 0.0123 - val_accuracy: 0.1346\n",
      "Epoch 12/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0116 - accuracy: 0.1723\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0116 - accuracy: 0.1723 - val_loss: 0.0124 - val_accuracy: 0.1394\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0113 - accuracy: 0.1985 - val_loss: 0.0121 - val_accuracy: 0.1374\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0112 - accuracy: 0.2017 - val_loss: 0.0122 - val_accuracy: 0.1415\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0111 - accuracy: 0.2040 - val_loss: 0.0121 - val_accuracy: 0.1415\n",
      "Epoch 16/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0110 - accuracy: 0.2100\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0110 - accuracy: 0.2102 - val_loss: 0.0121 - val_accuracy: 0.1431\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0109 - accuracy: 0.2137 - val_loss: 0.0121 - val_accuracy: 0.1419\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0109 - accuracy: 0.2168 - val_loss: 0.0121 - val_accuracy: 0.1410\n",
      "Epoch 19/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0109 - accuracy: 0.2159\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0109 - accuracy: 0.2164 - val_loss: 0.0121 - val_accuracy: 0.1419\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0109 - accuracy: 0.2165 - val_loss: 0.0121 - val_accuracy: 0.1417\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0109 - accuracy: 0.2130 - val_loss: 0.0121 - val_accuracy: 0.1419\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0109 - accuracy: 0.2146\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0109 - accuracy: 0.2146 - val_loss: 0.0121 - val_accuracy: 0.1419\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0109 - accuracy: 0.2142 - val_loss: 0.0121 - val_accuracy: 0.1421\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0109 - accuracy: 0.2175 - val_loss: 0.0121 - val_accuracy: 0.1419\n",
      "Epoch 25/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0109 - accuracy: 0.2152\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0109 - accuracy: 0.2155 - val_loss: 0.0121 - val_accuracy: 0.1421\n",
      "Epoch 00025: early stopping\n",
      "SEED:30 Fold:3\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2452 - accuracy: 0.0201 - val_loss: 0.0202 - val_accuracy: 0.0510\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0159 - accuracy: 0.0384 - val_loss: 0.0146 - val_accuracy: 0.0358\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0147 - accuracy: 0.0464 - val_loss: 0.0138 - val_accuracy: 0.0647\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0143 - accuracy: 0.0540 - val_loss: 0.0136 - val_accuracy: 0.0665\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0140 - accuracy: 0.0715 - val_loss: 0.0133 - val_accuracy: 0.0775\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0137 - accuracy: 0.0876 - val_loss: 0.0132 - val_accuracy: 0.0914\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0135 - accuracy: 0.0989 - val_loss: 0.0131 - val_accuracy: 0.0955\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0134 - accuracy: 0.1130 - val_loss: 0.0129 - val_accuracy: 0.1021\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0131 - accuracy: 0.1210 - val_loss: 0.0128 - val_accuracy: 0.1050\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.1299 - val_loss: 0.0128 - val_accuracy: 0.1144\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0132 - accuracy: 0.1247 - val_loss: 0.0131 - val_accuracy: 0.1126\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0130 - accuracy: 0.1273 - val_loss: 0.0127 - val_accuracy: 0.1233\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0127 - accuracy: 0.1391 - val_loss: 0.0131 - val_accuracy: 0.1194\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.1378 - val_loss: 0.0127 - val_accuracy: 0.1233\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0127 - accuracy: 0.1388 - val_loss: 0.0126 - val_accuracy: 0.1315\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0125 - accuracy: 0.1476 - val_loss: 0.0126 - val_accuracy: 0.1267\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0122 - accuracy: 0.1528 - val_loss: 0.0123 - val_accuracy: 0.1356\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0120 - accuracy: 0.1608 - val_loss: 0.0124 - val_accuracy: 0.1347\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0122 - accuracy: 0.1636 - val_loss: 0.0124 - val_accuracy: 0.1504\n",
      "Epoch 20/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0120 - accuracy: 0.1821\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0120 - accuracy: 0.1809 - val_loss: 0.0124 - val_accuracy: 0.1732\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0115 - accuracy: 0.1968 - val_loss: 0.0122 - val_accuracy: 0.1479\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0115 - accuracy: 0.1867 - val_loss: 0.0122 - val_accuracy: 0.1488\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0113 - accuracy: 0.1929 - val_loss: 0.0121 - val_accuracy: 0.1486\n",
      "Epoch 24/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0113 - accuracy: 0.1893\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0113 - accuracy: 0.1892 - val_loss: 0.0121 - val_accuracy: 0.1497\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1984 - val_loss: 0.0121 - val_accuracy: 0.1501\n",
      "Epoch 26/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1948 - val_loss: 0.0121 - val_accuracy: 0.1497\n",
      "Epoch 27/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0112 - accuracy: 0.1963\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1962 - val_loss: 0.0121 - val_accuracy: 0.1499\n",
      "Epoch 28/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1965 - val_loss: 0.0121 - val_accuracy: 0.1499\n",
      "Epoch 29/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1970 - val_loss: 0.0121 - val_accuracy: 0.1499\n",
      "Epoch 30/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0112 - accuracy: 0.1994\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0112 - accuracy: 0.2002 - val_loss: 0.0121 - val_accuracy: 0.1499\n",
      "Epoch 31/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1954 - val_loss: 0.0121 - val_accuracy: 0.1497\n",
      "Epoch 32/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1965 - val_loss: 0.0121 - val_accuracy: 0.1501\n",
      "Epoch 33/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0112 - accuracy: 0.1972\n",
      "Epoch 00033: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1974 - val_loss: 0.0121 - val_accuracy: 0.1501\n",
      "Epoch 34/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1952 - val_loss: 0.0121 - val_accuracy: 0.1504\n",
      "Epoch 35/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1981 - val_loss: 0.0121 - val_accuracy: 0.1501\n",
      "Epoch 36/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0112 - accuracy: 0.1953\n",
      "Epoch 00036: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1956 - val_loss: 0.0121 - val_accuracy: 0.1501\n",
      "Epoch 37/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1994 - val_loss: 0.0121 - val_accuracy: 0.1501\n",
      "Epoch 38/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1961 - val_loss: 0.0121 - val_accuracy: 0.1504\n",
      "Epoch 39/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0112 - accuracy: 0.1949\n",
      "Epoch 00039: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-10.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1951 - val_loss: 0.0121 - val_accuracy: 0.1504\n",
      "Epoch 40/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1953 - val_loss: 0.0121 - val_accuracy: 0.1492\n",
      "Epoch 41/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0112 - accuracy: 0.1974 - val_loss: 0.0121 - val_accuracy: 0.1501\n",
      "Epoch 42/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0112 - accuracy: 0.1966\n",
      "Epoch 00042: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-11.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1962 - val_loss: 0.0121 - val_accuracy: 0.1499\n",
      "Epoch 43/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1960 - val_loss: 0.0121 - val_accuracy: 0.1499\n",
      "Epoch 44/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0112 - accuracy: 0.1974 - val_loss: 0.0121 - val_accuracy: 0.1497\n",
      "Epoch 45/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0112 - accuracy: 0.1986\n",
      "Epoch 00045: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-12.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0112 - accuracy: 0.1991 - val_loss: 0.0121 - val_accuracy: 0.1497\n",
      "Epoch 46/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0112 - accuracy: 0.1961Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.1959 - val_loss: 0.0121 - val_accuracy: 0.1499\n",
      "Epoch 00046: early stopping\n",
      "SEED:30 Fold:4\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.2322 - accuracy: 0.0365 - val_loss: 0.0160 - val_accuracy: 0.0408\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0150 - accuracy: 0.0700 - val_loss: 0.0136 - val_accuracy: 0.0699\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0137 - accuracy: 0.0877 - val_loss: 0.0131 - val_accuracy: 0.1107\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0132 - accuracy: 0.1060 - val_loss: 0.0129 - val_accuracy: 0.1342\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0130 - accuracy: 0.1201 - val_loss: 0.0126 - val_accuracy: 0.1483\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0125 - accuracy: 0.1402 - val_loss: 0.0125 - val_accuracy: 0.1290\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0123 - accuracy: 0.1615 - val_loss: 0.0127 - val_accuracy: 0.1476\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0120 - accuracy: 0.1858 - val_loss: 0.0129 - val_accuracy: 0.1419\n",
      "Epoch 9/150\n",
      "128/138 [==========================>...] - ETA: 0s - loss: 0.0117 - accuracy: 0.1974\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0117 - accuracy: 0.1981 - val_loss: 0.0125 - val_accuracy: 0.1492\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0107 - accuracy: 0.2521 - val_loss: 0.0121 - val_accuracy: 0.1574\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0104 - accuracy: 0.2670 - val_loss: 0.0121 - val_accuracy: 0.1570\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0103 - accuracy: 0.2755 - val_loss: 0.0120 - val_accuracy: 0.1561\n",
      "Epoch 13/150\n",
      "129/138 [===========================>..] - ETA: 0s - loss: 0.0101 - accuracy: 0.2897\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0101 - accuracy: 0.2896 - val_loss: 0.0121 - val_accuracy: 0.1602\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0100 - accuracy: 0.3022 - val_loss: 0.0121 - val_accuracy: 0.1590\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0100 - accuracy: 0.3021 - val_loss: 0.0121 - val_accuracy: 0.1599\n",
      "Epoch 16/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0100 - accuracy: 0.3009\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0100 - accuracy: 0.3016 - val_loss: 0.0121 - val_accuracy: 0.1595\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0100 - accuracy: 0.3035 - val_loss: 0.0120 - val_accuracy: 0.1593\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0100 - accuracy: 0.3066 - val_loss: 0.0121 - val_accuracy: 0.1599\n",
      "Epoch 19/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0099 - accuracy: 0.3080\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0099 - accuracy: 0.3081 - val_loss: 0.0121 - val_accuracy: 0.1590\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0100 - accuracy: 0.2997 - val_loss: 0.0121 - val_accuracy: 0.1588\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0099 - accuracy: 0.3051 - val_loss: 0.0121 - val_accuracy: 0.1599\n",
      "Epoch 22/150\n",
      "129/138 [===========================>..] - ETA: 0s - loss: 0.0099 - accuracy: 0.2975\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0099 - accuracy: 0.2990 - val_loss: 0.0121 - val_accuracy: 0.1599\n",
      "Epoch 00022: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=58 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:58 Fold:0\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2358 - accuracy: 0.0260 - val_loss: 0.0171 - val_accuracy: 0.0298\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0153 - accuracy: 0.0555 - val_loss: 0.0140 - val_accuracy: 0.0834\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0142 - accuracy: 0.0715 - val_loss: 0.0138 - val_accuracy: 0.0961\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0137 - accuracy: 0.0960 - val_loss: 0.0131 - val_accuracy: 0.1178\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0133 - accuracy: 0.1115 - val_loss: 0.0130 - val_accuracy: 0.1314\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0132 - accuracy: 0.1209 - val_loss: 0.0128 - val_accuracy: 0.1121\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0128 - accuracy: 0.1344 - val_loss: 0.0127 - val_accuracy: 0.1478\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0125 - accuracy: 0.1447 - val_loss: 0.0182 - val_accuracy: 0.1239\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0128 - accuracy: 0.1438 - val_loss: 0.0125 - val_accuracy: 0.1305\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0122 - accuracy: 0.1619 - val_loss: 0.0123 - val_accuracy: 0.1337\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0120 - accuracy: 0.1794 - val_loss: 0.0122 - val_accuracy: 0.1362\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0117 - accuracy: 0.1903 - val_loss: 0.0121 - val_accuracy: 0.1474\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0119 - accuracy: 0.1974 - val_loss: 0.0126 - val_accuracy: 0.1456\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0115 - accuracy: 0.2075 - val_loss: 0.0123 - val_accuracy: 0.1519\n",
      "Epoch 15/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0112 - accuracy: 0.2212\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0112 - accuracy: 0.2216 - val_loss: 0.0123 - val_accuracy: 0.1850\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0106 - accuracy: 0.2606 - val_loss: 0.0121 - val_accuracy: 0.1551\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0104 - accuracy: 0.2665 - val_loss: 0.0121 - val_accuracy: 0.1567\n",
      "Epoch 18/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0103 - accuracy: 0.2683\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0103 - accuracy: 0.2676 - val_loss: 0.0121 - val_accuracy: 0.1563\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2787 - val_loss: 0.0121 - val_accuracy: 0.1576\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2808 - val_loss: 0.0121 - val_accuracy: 0.1583\n",
      "Epoch 21/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0102 - accuracy: 0.2805\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2811 - val_loss: 0.0121 - val_accuracy: 0.1585\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2845 - val_loss: 0.0121 - val_accuracy: 0.1576\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0102 - accuracy: 0.2823 - val_loss: 0.0121 - val_accuracy: 0.1574\n",
      "Epoch 24/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0102 - accuracy: 0.2821\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2822 - val_loss: 0.0121 - val_accuracy: 0.1579\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2839 - val_loss: 0.0121 - val_accuracy: 0.1583\n",
      "Epoch 26/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0103 - accuracy: 0.2817 - val_loss: 0.0121 - val_accuracy: 0.1583\n",
      "Epoch 27/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0102 - accuracy: 0.2820\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2824 - val_loss: 0.0121 - val_accuracy: 0.1588\n",
      "Epoch 28/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0102 - accuracy: 0.2823Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0102 - accuracy: 0.2824 - val_loss: 0.0121 - val_accuracy: 0.1576\n",
      "Epoch 00028: early stopping\n",
      "SEED:58 Fold:1\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.2634 - accuracy: 0.0177 - val_loss: 0.0202 - val_accuracy: 0.0169\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0166 - accuracy: 0.0339 - val_loss: 0.0143 - val_accuracy: 0.0169\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 2s 14ms/step - loss: 0.0146 - accuracy: 0.0389 - val_loss: 0.0137 - val_accuracy: 0.0643\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0144 - accuracy: 0.0497 - val_loss: 0.0136 - val_accuracy: 0.0643\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0140 - accuracy: 0.0629 - val_loss: 0.0133 - val_accuracy: 0.0961\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0137 - accuracy: 0.0785 - val_loss: 0.0131 - val_accuracy: 0.0715\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0137 - accuracy: 0.0850 - val_loss: 0.0131 - val_accuracy: 0.1116\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0134 - accuracy: 0.0974 - val_loss: 0.0130 - val_accuracy: 0.0927\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0133 - accuracy: 0.1022 - val_loss: 0.0131 - val_accuracy: 0.1201\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0132 - accuracy: 0.1031 - val_loss: 0.0129 - val_accuracy: 0.0975\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0130 - accuracy: 0.1117 - val_loss: 0.0127 - val_accuracy: 0.1089\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0130 - accuracy: 0.1145 - val_loss: 0.0135 - val_accuracy: 0.1053\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0130 - accuracy: 0.1183 - val_loss: 0.0127 - val_accuracy: 0.1059\n",
      "Epoch 14/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0128 - accuracy: 0.1217\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0128 - accuracy: 0.1214 - val_loss: 0.0133 - val_accuracy: 0.1119\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0127 - accuracy: 0.1280 - val_loss: 0.0126 - val_accuracy: 0.1157\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0125 - accuracy: 0.1309 - val_loss: 0.0125 - val_accuracy: 0.1157\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0125 - accuracy: 0.1330 - val_loss: 0.0125 - val_accuracy: 0.1180\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0125 - accuracy: 0.1322 - val_loss: 0.0125 - val_accuracy: 0.1180\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0124 - accuracy: 0.1340 - val_loss: 0.0125 - val_accuracy: 0.1187\n",
      "Epoch 20/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0124 - accuracy: 0.1328\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0124 - accuracy: 0.1332 - val_loss: 0.0124 - val_accuracy: 0.1187\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0124 - accuracy: 0.1318 - val_loss: 0.0124 - val_accuracy: 0.1189\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0123 - accuracy: 0.1351 - val_loss: 0.0124 - val_accuracy: 0.1189\n",
      "Epoch 23/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0123 - accuracy: 0.1343\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0123 - accuracy: 0.1348 - val_loss: 0.0124 - val_accuracy: 0.1194\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0123 - accuracy: 0.1360 - val_loss: 0.0124 - val_accuracy: 0.1194\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0123 - accuracy: 0.1354 - val_loss: 0.0124 - val_accuracy: 0.1198\n",
      "Epoch 26/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0123 - accuracy: 0.1358\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0123 - accuracy: 0.1358 - val_loss: 0.0124 - val_accuracy: 0.1196\n",
      "Epoch 27/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0123 - accuracy: 0.1345 - val_loss: 0.0124 - val_accuracy: 0.1201\n",
      "Epoch 28/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0123 - accuracy: 0.1384 - val_loss: 0.0124 - val_accuracy: 0.1198\n",
      "Epoch 29/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0123 - accuracy: 0.1366\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0123 - accuracy: 0.1364 - val_loss: 0.0124 - val_accuracy: 0.1196\n",
      "Epoch 30/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0123 - accuracy: 0.1371 - val_loss: 0.0124 - val_accuracy: 0.1198\n",
      "Epoch 31/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0123 - accuracy: 0.1383 - val_loss: 0.0124 - val_accuracy: 0.1196\n",
      "Epoch 32/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0123 - accuracy: 0.1364\n",
      "Epoch 00032: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0123 - accuracy: 0.1366 - val_loss: 0.0124 - val_accuracy: 0.1198\n",
      "Epoch 00032: early stopping\n",
      "SEED:58 Fold:2\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2908 - accuracy: 0.0223 - val_loss: 0.0260 - val_accuracy: 0.0408\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0168 - accuracy: 0.0464 - val_loss: 0.0147 - val_accuracy: 0.0415\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0139 - accuracy: 0.0575 - val_loss: 0.0134 - val_accuracy: 0.0955\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0131 - accuracy: 0.0835 - val_loss: 0.0130 - val_accuracy: 0.1189\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0127 - accuracy: 0.1064 - val_loss: 0.0127 - val_accuracy: 0.1365\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0125 - accuracy: 0.1243 - val_loss: 0.0124 - val_accuracy: 0.1208\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0121 - accuracy: 0.1386 - val_loss: 0.0123 - val_accuracy: 0.1426\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0119 - accuracy: 0.1556 - val_loss: 0.0123 - val_accuracy: 0.1376\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0116 - accuracy: 0.1711 - val_loss: 0.0122 - val_accuracy: 0.1431\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0114 - accuracy: 0.1861 - val_loss: 0.0123 - val_accuracy: 0.1809\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0111 - accuracy: 0.2056 - val_loss: 0.0122 - val_accuracy: 0.1499\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0109 - accuracy: 0.2232\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0109 - accuracy: 0.2232 - val_loss: 0.0122 - val_accuracy: 0.1572\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0102 - accuracy: 0.2641 - val_loss: 0.0120 - val_accuracy: 0.1609\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0100 - accuracy: 0.2759 - val_loss: 0.0120 - val_accuracy: 0.1661\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0099 - accuracy: 0.2851 - val_loss: 0.0120 - val_accuracy: 0.1663\n",
      "Epoch 16/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0098 - accuracy: 0.2924\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0098 - accuracy: 0.2922 - val_loss: 0.0120 - val_accuracy: 0.1722\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0097 - accuracy: 0.3057 - val_loss: 0.0120 - val_accuracy: 0.1668\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0097 - accuracy: 0.3059 - val_loss: 0.0120 - val_accuracy: 0.1666\n",
      "Epoch 19/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0097 - accuracy: 0.3066\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0097 - accuracy: 0.3056 - val_loss: 0.0120 - val_accuracy: 0.1677\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0096 - accuracy: 0.3125 - val_loss: 0.0121 - val_accuracy: 0.1672\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0096 - accuracy: 0.3110 - val_loss: 0.0121 - val_accuracy: 0.1672\n",
      "Epoch 22/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0096 - accuracy: 0.3059\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0096 - accuracy: 0.3066 - val_loss: 0.0121 - val_accuracy: 0.1675\n",
      "Epoch 23/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0097 - accuracy: 0.3076Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0096 - accuracy: 0.3077 - val_loss: 0.0121 - val_accuracy: 0.1677\n",
      "Epoch 00023: early stopping\n",
      "SEED:58 Fold:3\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 14ms/step - loss: 0.2887 - accuracy: 0.0345 - val_loss: 0.0224 - val_accuracy: 0.0123\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0166 - accuracy: 0.0504 - val_loss: 0.0144 - val_accuracy: 0.0458\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0138 - accuracy: 0.0587 - val_loss: 0.0133 - val_accuracy: 0.0624\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0130 - accuracy: 0.0862 - val_loss: 0.0129 - val_accuracy: 0.0957\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0126 - accuracy: 0.1084 - val_loss: 0.0127 - val_accuracy: 0.0991\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0124 - accuracy: 0.1174 - val_loss: 0.0125 - val_accuracy: 0.1182\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0122 - accuracy: 0.1340 - val_loss: 0.0124 - val_accuracy: 0.1232\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0120 - accuracy: 0.1398 - val_loss: 0.0123 - val_accuracy: 0.1483\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0117 - accuracy: 0.1502 - val_loss: 0.0122 - val_accuracy: 0.1487\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0115 - accuracy: 0.1632 - val_loss: 0.0121 - val_accuracy: 0.1424\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0113 - accuracy: 0.1774 - val_loss: 0.0122 - val_accuracy: 0.1453\n",
      "Epoch 12/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0111 - accuracy: 0.1904\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0111 - accuracy: 0.1903 - val_loss: 0.0122 - val_accuracy: 0.1474\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0106 - accuracy: 0.2205 - val_loss: 0.0119 - val_accuracy: 0.1565\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0104 - accuracy: 0.2314 - val_loss: 0.0119 - val_accuracy: 0.1581\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0103 - accuracy: 0.2382 - val_loss: 0.0119 - val_accuracy: 0.1574\n",
      "Epoch 16/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0103 - accuracy: 0.2459\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0103 - accuracy: 0.2457 - val_loss: 0.0119 - val_accuracy: 0.1579\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0101 - accuracy: 0.2523 - val_loss: 0.0119 - val_accuracy: 0.1588\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0101 - accuracy: 0.2516 - val_loss: 0.0119 - val_accuracy: 0.1590\n",
      "Epoch 19/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0101 - accuracy: 0.2558\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0101 - accuracy: 0.2556 - val_loss: 0.0119 - val_accuracy: 0.1601\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0101 - accuracy: 0.2550 - val_loss: 0.0119 - val_accuracy: 0.1599\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0101 - accuracy: 0.2532 - val_loss: 0.0119 - val_accuracy: 0.1599\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0101 - accuracy: 0.2538\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0101 - accuracy: 0.2538 - val_loss: 0.0119 - val_accuracy: 0.1592\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0101 - accuracy: 0.2543 - val_loss: 0.0119 - val_accuracy: 0.1601\n",
      "Epoch 24/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0101 - accuracy: 0.2528Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0101 - accuracy: 0.2543 - val_loss: 0.0119 - val_accuracy: 0.1592\n",
      "Epoch 00024: early stopping\n",
      "SEED:58 Fold:4\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.2934 - accuracy: 0.0157 - val_loss: 0.0207 - val_accuracy: 0.0050\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0168 - accuracy: 0.0377 - val_loss: 0.0145 - val_accuracy: 0.0342\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0139 - accuracy: 0.0555 - val_loss: 0.0135 - val_accuracy: 0.0888\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0132 - accuracy: 0.0751 - val_loss: 0.0131 - val_accuracy: 0.1091\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.1010 - val_loss: 0.0127 - val_accuracy: 0.0916\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0126 - accuracy: 0.1097 - val_loss: 0.0125 - val_accuracy: 0.1353\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0124 - accuracy: 0.1240 - val_loss: 0.0124 - val_accuracy: 0.1282\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0122 - accuracy: 0.1385 - val_loss: 0.0123 - val_accuracy: 0.1303\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0121 - accuracy: 0.1449 - val_loss: 0.0122 - val_accuracy: 0.1339\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0118 - accuracy: 0.1517 - val_loss: 0.0121 - val_accuracy: 0.1387\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0117 - accuracy: 0.1656 - val_loss: 0.0120 - val_accuracy: 0.1460\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0115 - accuracy: 0.1746 - val_loss: 0.0121 - val_accuracy: 0.1426\n",
      "Epoch 13/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0113 - accuracy: 0.1897\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0113 - accuracy: 0.1897 - val_loss: 0.0121 - val_accuracy: 0.1779\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0109 - accuracy: 0.2218 - val_loss: 0.0119 - val_accuracy: 0.1581\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0107 - accuracy: 0.2212 - val_loss: 0.0119 - val_accuracy: 0.1595\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0107 - accuracy: 0.2243 - val_loss: 0.0119 - val_accuracy: 0.1601\n",
      "Epoch 17/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0106 - accuracy: 0.2269\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0106 - accuracy: 0.2268 - val_loss: 0.0119 - val_accuracy: 0.1601\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0105 - accuracy: 0.2352 - val_loss: 0.0119 - val_accuracy: 0.1608\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0105 - accuracy: 0.2299 - val_loss: 0.0119 - val_accuracy: 0.1617\n",
      "Epoch 20/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0105 - accuracy: 0.2326\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0105 - accuracy: 0.2324 - val_loss: 0.0119 - val_accuracy: 0.1610\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0105 - accuracy: 0.2330 - val_loss: 0.0119 - val_accuracy: 0.1613\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0105 - accuracy: 0.2329 - val_loss: 0.0119 - val_accuracy: 0.1613\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0105 - accuracy: 0.2336\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0105 - accuracy: 0.2336 - val_loss: 0.0119 - val_accuracy: 0.1617\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0105 - accuracy: 0.2350 - val_loss: 0.0119 - val_accuracy: 0.1617\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0105 - accuracy: 0.2344Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0105 - accuracy: 0.2344 - val_loss: 0.0119 - val_accuracy: 0.1620\n",
      "Epoch 00025: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=3255 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:3255 Fold:0\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 3s 21ms/step - loss: 0.2885 - accuracy: 0.0234 - val_loss: 0.0215 - val_accuracy: 0.0105\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0167 - accuracy: 0.0413 - val_loss: 0.0145 - val_accuracy: 0.0355\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0138 - accuracy: 0.0508 - val_loss: 0.0134 - val_accuracy: 0.0551\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0131 - accuracy: 0.0773 - val_loss: 0.0129 - val_accuracy: 0.1096\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0127 - accuracy: 0.1033 - val_loss: 0.0125 - val_accuracy: 0.1130\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0125 - accuracy: 0.1195 - val_loss: 0.0125 - val_accuracy: 0.1424\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0123 - accuracy: 0.1255 - val_loss: 0.0123 - val_accuracy: 0.1264\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0121 - accuracy: 0.1392 - val_loss: 0.0122 - val_accuracy: 0.1253\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0119 - accuracy: 0.1474 - val_loss: 0.0123 - val_accuracy: 0.1360\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0117 - accuracy: 0.1588 - val_loss: 0.0121 - val_accuracy: 0.1752\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0115 - accuracy: 0.1747 - val_loss: 0.0120 - val_accuracy: 0.1476\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0114 - accuracy: 0.1809 - val_loss: 0.0121 - val_accuracy: 0.1476\n",
      "Epoch 13/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0112 - accuracy: 0.1928\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0112 - accuracy: 0.1921 - val_loss: 0.0121 - val_accuracy: 0.1476\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0107 - accuracy: 0.2134 - val_loss: 0.0118 - val_accuracy: 0.1540\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0106 - accuracy: 0.2168 - val_loss: 0.0118 - val_accuracy: 0.1526\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0105 - accuracy: 0.2254 - val_loss: 0.0118 - val_accuracy: 0.1542\n",
      "Epoch 17/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0105 - accuracy: 0.2284\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0105 - accuracy: 0.2283 - val_loss: 0.0118 - val_accuracy: 0.1538\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0104 - accuracy: 0.2354 - val_loss: 0.0118 - val_accuracy: 0.1547\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 2s 14ms/step - loss: 0.0104 - accuracy: 0.2345 - val_loss: 0.0118 - val_accuracy: 0.1547\n",
      "Epoch 20/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0104 - accuracy: 0.2374\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0104 - accuracy: 0.2374 - val_loss: 0.0118 - val_accuracy: 0.1556\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0104 - accuracy: 0.2389 - val_loss: 0.0118 - val_accuracy: 0.1544\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0104 - accuracy: 0.2393 - val_loss: 0.0118 - val_accuracy: 0.1544\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0104 - accuracy: 0.2372\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0104 - accuracy: 0.2372 - val_loss: 0.0118 - val_accuracy: 0.1544\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0104 - accuracy: 0.2402 - val_loss: 0.0118 - val_accuracy: 0.1549\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0104 - accuracy: 0.2364 - val_loss: 0.0118 - val_accuracy: 0.1547\n",
      "Epoch 26/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0104 - accuracy: 0.2412\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0104 - accuracy: 0.2407 - val_loss: 0.0118 - val_accuracy: 0.1542\n",
      "Epoch 00026: early stopping\n",
      "SEED:3255 Fold:1\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.2816 - accuracy: 0.0208 - val_loss: 0.0228 - val_accuracy: 0.0476\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0168 - accuracy: 0.0395 - val_loss: 0.0152 - val_accuracy: 0.0271\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0144 - accuracy: 0.0478 - val_loss: 0.0135 - val_accuracy: 0.0488\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0140 - accuracy: 0.0564 - val_loss: 0.0132 - val_accuracy: 0.0508\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0139 - accuracy: 0.0688 - val_loss: 0.0132 - val_accuracy: 0.0727\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0136 - accuracy: 0.0794 - val_loss: 0.0129 - val_accuracy: 0.0795\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0132 - accuracy: 0.0886 - val_loss: 0.0128 - val_accuracy: 0.0946\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0130 - accuracy: 0.1020 - val_loss: 0.0126 - val_accuracy: 0.1082\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0128 - accuracy: 0.1134 - val_loss: 0.0126 - val_accuracy: 0.1069\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0127 - accuracy: 0.1234 - val_loss: 0.0125 - val_accuracy: 0.1157\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0125 - accuracy: 0.1300 - val_loss: 0.0124 - val_accuracy: 0.1280\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0123 - accuracy: 0.1408 - val_loss: 0.0123 - val_accuracy: 0.1577\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0122 - accuracy: 0.1445 - val_loss: 0.0123 - val_accuracy: 0.1251\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0121 - accuracy: 0.1601\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0121 - accuracy: 0.1601 - val_loss: 0.0124 - val_accuracy: 0.1285\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0117 - accuracy: 0.1649 - val_loss: 0.0122 - val_accuracy: 0.1312\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0116 - accuracy: 0.1709 - val_loss: 0.0121 - val_accuracy: 0.1340\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0115 - accuracy: 0.1744 - val_loss: 0.0121 - val_accuracy: 0.1331\n",
      "Epoch 18/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0114 - accuracy: 0.1788\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0114 - accuracy: 0.1788 - val_loss: 0.0121 - val_accuracy: 0.1340\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0114 - accuracy: 0.1840 - val_loss: 0.0121 - val_accuracy: 0.1340\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0114 - accuracy: 0.1823 - val_loss: 0.0121 - val_accuracy: 0.1340\n",
      "Epoch 21/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0114 - accuracy: 0.1825\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0114 - accuracy: 0.1829 - val_loss: 0.0121 - val_accuracy: 0.1337\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0114 - accuracy: 0.1814 - val_loss: 0.0121 - val_accuracy: 0.1347\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0114 - accuracy: 0.1826 - val_loss: 0.0121 - val_accuracy: 0.1344\n",
      "Epoch 24/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0113 - accuracy: 0.1818\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0113 - accuracy: 0.1818 - val_loss: 0.0121 - val_accuracy: 0.1337\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0114 - accuracy: 0.1841 - val_loss: 0.0121 - val_accuracy: 0.1335\n",
      "Epoch 26/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0113 - accuracy: 0.1847 - val_loss: 0.0121 - val_accuracy: 0.1340\n",
      "Epoch 27/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0113 - accuracy: 0.1835\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0113 - accuracy: 0.1834 - val_loss: 0.0121 - val_accuracy: 0.1337\n",
      "Epoch 28/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0113 - accuracy: 0.1799Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0113 - accuracy: 0.1799 - val_loss: 0.0121 - val_accuracy: 0.1333\n",
      "Epoch 00028: early stopping\n",
      "SEED:3255 Fold:2\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2602 - accuracy: 0.0289 - val_loss: 0.0173 - val_accuracy: 0.0410\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0160 - accuracy: 0.0568 - val_loss: 0.0140 - val_accuracy: 0.0565\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0144 - accuracy: 0.0693 - val_loss: 0.0137 - val_accuracy: 0.0747\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0140 - accuracy: 0.0804 - val_loss: 0.0135 - val_accuracy: 0.1128\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0141 - accuracy: 0.0917 - val_loss: 0.0134 - val_accuracy: 0.0868\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0136 - accuracy: 0.1038 - val_loss: 0.0133 - val_accuracy: 0.1298\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0135 - accuracy: 0.1107 - val_loss: 0.0133 - val_accuracy: 0.1023\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0135 - accuracy: 0.1172 - val_loss: 0.0134 - val_accuracy: 0.1139\n",
      "Epoch 9/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0147 - accuracy: 0.1040\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0146 - accuracy: 0.1035 - val_loss: 0.0134 - val_accuracy: 0.0995\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0134 - accuracy: 0.1103 - val_loss: 0.0132 - val_accuracy: 0.1080\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0133 - accuracy: 0.1128 - val_loss: 0.0131 - val_accuracy: 0.1066\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0133 - accuracy: 0.1145 - val_loss: 0.0131 - val_accuracy: 0.1064\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0132 - accuracy: 0.1135 - val_loss: 0.0130 - val_accuracy: 0.1064\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0131 - accuracy: 0.1145 - val_loss: 0.0129 - val_accuracy: 0.1075\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0130 - accuracy: 0.1182 - val_loss: 0.0129 - val_accuracy: 0.1098\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0130 - accuracy: 0.1176 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 17/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0130 - accuracy: 0.1194\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0130 - accuracy: 0.1194 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0130 - accuracy: 0.1187 - val_loss: 0.0129 - val_accuracy: 0.1100\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1206 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 20/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1177\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1183 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1169 - val_loss: 0.0129 - val_accuracy: 0.1093\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1190 - val_loss: 0.0129 - val_accuracy: 0.1098\n",
      "Epoch 23/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1216\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1213 - val_loss: 0.0129 - val_accuracy: 0.1096\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1208 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1205 - val_loss: 0.0129 - val_accuracy: 0.1096\n",
      "Epoch 26/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1199\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1203 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 27/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1213 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 28/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1196 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 29/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1197\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1202 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 30/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0129 - accuracy: 0.1201 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 31/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.1192 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 32/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1199 - val_loss: 0.0128 - val_accuracy: 0.1089\n",
      "Epoch 33/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1204 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 34/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1190 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 35/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1179\n",
      "Epoch 00035: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-10.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1180 - val_loss: 0.0128 - val_accuracy: 0.1089\n",
      "Epoch 36/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0129 - accuracy: 0.1218 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 37/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1194 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 38/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1220\n",
      "Epoch 00038: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-11.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1220 - val_loss: 0.0129 - val_accuracy: 0.1098\n",
      "Epoch 39/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1202 - val_loss: 0.0128 - val_accuracy: 0.1096\n",
      "Epoch 40/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.1211 - val_loss: 0.0129 - val_accuracy: 0.1093\n",
      "Epoch 41/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0129 - accuracy: 0.1185\n",
      "Epoch 00041: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-12.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0129 - accuracy: 0.1184 - val_loss: 0.0128 - val_accuracy: 0.1087\n",
      "Epoch 42/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1205 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 43/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1202 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 44/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1204\n",
      "Epoch 00044: ReduceLROnPlateau reducing learning rate to 1.0000001044244145e-13.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1207 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 45/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1189 - val_loss: 0.0129 - val_accuracy: 0.1093\n",
      "Epoch 46/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1201 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 47/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1203\n",
      "Epoch 00047: ReduceLROnPlateau reducing learning rate to 1.0000001179769417e-14.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1213 - val_loss: 0.0128 - val_accuracy: 0.1093\n",
      "Epoch 48/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0130 - accuracy: 0.1208 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 49/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1224 - val_loss: 0.0129 - val_accuracy: 0.1089\n",
      "Epoch 50/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0129 - accuracy: 0.1174\n",
      "Epoch 00050: ReduceLROnPlateau reducing learning rate to 1.0000001518582595e-15.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1174 - val_loss: 0.0128 - val_accuracy: 0.1093\n",
      "Epoch 51/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1195 - val_loss: 0.0129 - val_accuracy: 0.1093\n",
      "Epoch 52/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0129 - accuracy: 0.1168 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 53/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1168\n",
      "Epoch 00053: ReduceLROnPlateau reducing learning rate to 1.0000001095066122e-16.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1172 - val_loss: 0.0129 - val_accuracy: 0.1098\n",
      "Epoch 54/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1215 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 55/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1218 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 56/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1211\n",
      "Epoch 00056: ReduceLROnPlateau reducing learning rate to 1.0000000830368326e-17.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1211 - val_loss: 0.0129 - val_accuracy: 0.1091\n",
      "Epoch 57/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0129 - accuracy: 0.1194Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1190 - val_loss: 0.0129 - val_accuracy: 0.1098\n",
      "Epoch 00057: early stopping\n",
      "SEED:3255 Fold:3\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.2400 - accuracy: 0.0306 - val_loss: 0.0196 - val_accuracy: 0.0567\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0162 - accuracy: 0.0507 - val_loss: 0.0148 - val_accuracy: 0.0504\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0144 - accuracy: 0.0633 - val_loss: 0.0136 - val_accuracy: 0.0966\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0139 - accuracy: 0.0755 - val_loss: 0.0133 - val_accuracy: 0.0832\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0135 - accuracy: 0.0923 - val_loss: 0.0130 - val_accuracy: 0.0973\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0133 - accuracy: 0.1047 - val_loss: 0.0129 - val_accuracy: 0.0882\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.1125 - val_loss: 0.0128 - val_accuracy: 0.0989\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0127 - accuracy: 0.1197 - val_loss: 0.0125 - val_accuracy: 0.1404\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0124 - accuracy: 0.1317 - val_loss: 0.0124 - val_accuracy: 0.1274\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0123 - accuracy: 0.1425 - val_loss: 0.0124 - val_accuracy: 0.1187\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0120 - accuracy: 0.1522 - val_loss: 0.0122 - val_accuracy: 0.1388\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0118 - accuracy: 0.1691 - val_loss: 0.0121 - val_accuracy: 0.1335\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0116 - accuracy: 0.1782 - val_loss: 0.0121 - val_accuracy: 0.1401\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0114 - accuracy: 0.1997 - val_loss: 0.0123 - val_accuracy: 0.1324\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0111 - accuracy: 0.2166 - val_loss: 0.0121 - val_accuracy: 0.1511\n",
      "Epoch 16/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0109 - accuracy: 0.2291\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0109 - accuracy: 0.2285 - val_loss: 0.0121 - val_accuracy: 0.1447\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0103 - accuracy: 0.2671 - val_loss: 0.0119 - val_accuracy: 0.1515\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0100 - accuracy: 0.2832 - val_loss: 0.0119 - val_accuracy: 0.1517\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0099 - accuracy: 0.2911 - val_loss: 0.0119 - val_accuracy: 0.1542\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0098 - accuracy: 0.2936\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0098 - accuracy: 0.2936 - val_loss: 0.0119 - val_accuracy: 0.1684\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0097 - accuracy: 0.3100 - val_loss: 0.0119 - val_accuracy: 0.1595\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0097 - accuracy: 0.3105 - val_loss: 0.0119 - val_accuracy: 0.1572\n",
      "Epoch 23/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0097 - accuracy: 0.3104\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0097 - accuracy: 0.3104 - val_loss: 0.0119 - val_accuracy: 0.1577\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0097 - accuracy: 0.3105 - val_loss: 0.0119 - val_accuracy: 0.1588\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0097 - accuracy: 0.3107 - val_loss: 0.0119 - val_accuracy: 0.1577\n",
      "Epoch 26/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0097 - accuracy: 0.3117\n",
      "Epoch 00026: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0097 - accuracy: 0.3126 - val_loss: 0.0119 - val_accuracy: 0.1570\n",
      "Epoch 27/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0097 - accuracy: 0.3106 - val_loss: 0.0119 - val_accuracy: 0.1565\n",
      "Epoch 28/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0097 - accuracy: 0.3113 - val_loss: 0.0119 - val_accuracy: 0.1572\n",
      "Epoch 29/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0097 - accuracy: 0.3106\n",
      "Epoch 00029: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0097 - accuracy: 0.3104 - val_loss: 0.0119 - val_accuracy: 0.1579\n",
      "Epoch 00029: early stopping\n",
      "SEED:3255 Fold:4\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.2900 - accuracy: 0.0211 - val_loss: 0.0222 - val_accuracy: 0.0162\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0167 - accuracy: 0.0448 - val_loss: 0.0146 - val_accuracy: 0.0287\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0138 - accuracy: 0.0633 - val_loss: 0.0135 - val_accuracy: 0.1036\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0130 - accuracy: 0.0927 - val_loss: 0.0129 - val_accuracy: 0.0986\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0125 - accuracy: 0.1132 - val_loss: 0.0125 - val_accuracy: 0.1146\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0122 - accuracy: 0.1321 - val_loss: 0.0123 - val_accuracy: 0.1241\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0119 - accuracy: 0.1567 - val_loss: 0.0123 - val_accuracy: 0.1342\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0116 - accuracy: 0.1783 - val_loss: 0.0123 - val_accuracy: 0.1437\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0113 - accuracy: 0.1939 - val_loss: 0.0122 - val_accuracy: 0.1399\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0111 - accuracy: 0.2065 - val_loss: 0.0122 - val_accuracy: 0.1431\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0108 - accuracy: 0.2311 - val_loss: 0.0122 - val_accuracy: 0.1467\n",
      "Epoch 12/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0104 - accuracy: 0.2514\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0104 - accuracy: 0.2516 - val_loss: 0.0122 - val_accuracy: 0.1640\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0096 - accuracy: 0.3287 - val_loss: 0.0121 - val_accuracy: 0.1601\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0093 - accuracy: 0.3430 - val_loss: 0.0121 - val_accuracy: 0.1718\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0092 - accuracy: 0.3656 - val_loss: 0.0121 - val_accuracy: 0.1708\n",
      "Epoch 16/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0091 - accuracy: 0.3797\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0091 - accuracy: 0.3793 - val_loss: 0.0122 - val_accuracy: 0.1661\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0089 - accuracy: 0.3914 - val_loss: 0.0122 - val_accuracy: 0.1658\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0089 - accuracy: 0.3929 - val_loss: 0.0122 - val_accuracy: 0.1647\n",
      "Epoch 19/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0089 - accuracy: 0.3952\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0089 - accuracy: 0.3950 - val_loss: 0.0122 - val_accuracy: 0.1651\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0089 - accuracy: 0.3994 - val_loss: 0.0122 - val_accuracy: 0.1640\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0089 - accuracy: 0.4001 - val_loss: 0.0122 - val_accuracy: 0.1647\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0089 - accuracy: 0.3951\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.3951 - val_loss: 0.0122 - val_accuracy: 0.1645\n",
      "Epoch 23/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0089 - accuracy: 0.3995Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0089 - accuracy: 0.3987 - val_loss: 0.0122 - val_accuracy: 0.1645\n",
      "Epoch 00023: early stopping\n"
     ]
    }
   ],
   "source": [
    "y_train_non=y_train_non.iloc[:,y_train_non.columns!='sig_id']\n",
    "final_y=pd.concat([y_train,y_train_non],1)\n",
    "final_pred=pd.DataFrame(np.zeros((submission.shape[0], final_y.shape[1])),columns=final_y.columns)\n",
    "\n",
    "for seed in [30,58,3255]:\n",
    "    for n,(tr,te) in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True).split(X_train,y_train)):\n",
    "        print(\"SEED:\"+str(seed)+\" Fold:\"+str(n))\n",
    "        \n",
    "\n",
    "        units=hps[seed][n]['units']\n",
    "        activations=hps[seed][n]['activations']\n",
    "        dropouts=hps[seed][n]['dropouts']\n",
    "        \n",
    "        model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, final_y.shape[1])\n",
    "        \n",
    "\n",
    "        model.fit(X_train.values[tr],\n",
    "                 final_y.values[tr],\n",
    "                 batch_size=128,\n",
    "                 epochs=150,\n",
    "                 validation_data=(X_train.values[te],final_y.values[te]),\n",
    "                 verbose=1,\n",
    "                  callbacks=[\n",
    "                      tf.keras.callbacks.ReduceLROnPlateau(\n",
    "                          monitor='val_loss', \n",
    "                          factor=0.1, \n",
    "                          patience=3,\n",
    "                          epsilon = 1e-4, \n",
    "                          mode = 'min',\n",
    "                          verbose=1\n",
    "                      )\n",
    "                      ,\n",
    "\n",
    "                      tf.keras.callbacks.EarlyStopping(\n",
    "                          monitor='val_loss',\n",
    "                          min_delta=0,\n",
    "                          patience=10,\n",
    "                          mode='auto',\n",
    "                          verbose=1,\n",
    "                          baseline=None,\n",
    "                          restore_best_weights=True\n",
    "                      )\n",
    "                  ]\n",
    "                 )\n",
    "        model.save(f'pSEED{seed}FOLD{n}.h5')\n",
    "        model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, final_y.shape[1])\n",
    "        model.load_weights(f'pSEED{seed}FOLD{n}.h5')\n",
    "        final_pred+=model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:17:05.931219Z",
     "iopub.status.busy": "2020-11-26T03:17:05.930225Z",
     "iopub.status.idle": "2020-11-26T03:17:05.934103Z",
     "shell.execute_reply": "2020-11-26T03:17:05.933525Z"
    },
    "papermill": {
     "duration": 3.433849,
     "end_time": "2020-11-26T03:17:05.934254",
     "exception": false,
     "start_time": "2020-11-26T03:17:02.500405",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_pred=final_pred.loc[:,submission.columns[1:]]/15"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "papermill": {
     "duration": 3.670686,
     "end_time": "2020-11-26T03:17:13.035160",
     "exception": false,
     "start_time": "2020-11-26T03:17:09.364474",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<h1>TRANSFER LEARNING START</h1>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:17:19.877407Z",
     "iopub.status.busy": "2020-11-26T03:17:19.875154Z",
     "iopub.status.idle": "2020-11-26T03:17:19.878204Z",
     "shell.execute_reply": "2020-11-26T03:17:19.878769Z"
    },
    "papermill": {
     "duration": 3.408161,
     "end_time": "2020-11-26T03:17:19.878922",
     "exception": false,
     "start_time": "2020-11-26T03:17:16.470761",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "############################### HPS for transfer learning ##############################################\n",
    "hps={\n",
    "    30:[\n",
    "    {\n",
    "        \"units\":[512,1187, 2048, 1971],\n",
    "        \"activations\":['elu', 'swish', 'selu','swish'],\n",
    "        \"dropouts\":[0.28,0.26,0.47,0.6]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[1127,532, 1488, 1219],\n",
    "        \"activations\":['swish', 'selu','elu','swish'],\n",
    "        \"dropouts\":[0.57,0.0,0.16,0.44]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[1123,640, 512],\n",
    "        \"activations\":['elu','swish','elu'],\n",
    "        \"dropouts\":[0.14,0.64,0.0]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[1110,1248,588, 971],\n",
    "        \"activations\":['selu','selu','elu','selu'],\n",
    "        \"dropouts\":[0.29,0.61,0.64,0.41]\n",
    "    },\n",
    "    {\n",
    "        \"units\":[1687,1494,1513],\n",
    "        \"activations\":['elu','swish','elu'],\n",
    "        \"dropouts\":[0.17,0.02,0.66]\n",
    "    }    \n",
    "    ],\n",
    "    58:[\n",
    "        {\n",
    "            \"units\":[1395,1309],\n",
    "            \"activations\":['selu','elu'],\n",
    "            \"dropouts\":[0.51,0.36]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[897,1321],\n",
    "            \"activations\":['elu', 'swish'],\n",
    "            \"dropouts\":[0.59,0.63]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1235,841],\n",
    "            \"activations\":['selu', 'elu'],\n",
    "            \"dropouts\":[0.69,0.05]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1628,1272,512],\n",
    "            \"activations\":['elu', 'swish', 'elu'],\n",
    "            \"dropouts\":[0.63,0.44,0.0]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[646,1056,1827,774,1463],\n",
    "            \"activations\":['selu','elu','swish','selu','selu'],\n",
    "            \"dropouts\":[0.51,0.62,0.47,0.35,0.65]\n",
    "        }\n",
    "    ],\n",
    "    3255:[\n",
    "        {\n",
    "            \"units\":[1148,1108,512,512],\n",
    "            \"activations\":['selu', 'swish', 'elu', 'elu'],\n",
    "            \"dropouts\":[0.28,0.69,0.0,0.0]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1060,778,1430],\n",
    "            \"activations\":['elu', 'selu', 'elu'],\n",
    "            \"dropouts\":[0.68,0.44,0.66]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1742,1453,1871,1655],\n",
    "            \"activations\":['selu', 'swish', 'swish', 'selu'],\n",
    "            \"dropouts\":[0.46,0.57,0.5,0.61]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[1788,1817,512,512,512,512],\n",
    "            \"activations\":[ 'swish', 'elu', 'elu','elu','elu','elu'],\n",
    "            \"dropouts\":[0.68,0.54,0.0,0.0,0.0,0.0]\n",
    "        },\n",
    "        {\n",
    "            \"units\":[907,764],\n",
    "            \"activations\":[  'elu', 'elu'],\n",
    "            \"dropouts\":[0.47,0.38]\n",
    "        }\n",
    "    ]\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:17:27.056756Z",
     "iopub.status.busy": "2020-11-26T03:17:27.054091Z",
     "iopub.status.idle": "2020-11-26T03:17:27.063894Z",
     "shell.execute_reply": "2020-11-26T03:17:27.062975Z"
    },
    "papermill": {
     "duration": 3.531622,
     "end_time": "2020-11-26T03:17:27.064075",
     "exception": false,
     "start_time": "2020-11-26T03:17:23.532453",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "final_pred=submission.iloc[:,1:].copy()\n",
    "final_pred.loc[:,:]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:17:35.323430Z",
     "iopub.status.busy": "2020-11-26T03:17:35.322274Z",
     "iopub.status.idle": "2020-11-26T03:30:52.265673Z",
     "shell.execute_reply": "2020-11-26T03:30:52.266602Z"
    },
    "papermill": {
     "duration": 801.163664,
     "end_time": "2020-11-26T03:30:52.266805",
     "exception": false,
     "start_time": "2020-11-26T03:17:31.103141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=30 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:30 Fold:0\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.2517 - accuracy: 0.0019 - val_loss: 0.0134 - val_accuracy: 0.0030\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0110 - accuracy: 0.0033 - val_loss: 0.0091 - val_accuracy: 0.0050\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0093 - accuracy: 0.0042 - val_loss: 0.0086 - val_accuracy: 0.0062\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0090 - accuracy: 0.0048 - val_loss: 0.0086 - val_accuracy: 0.0059\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0054 - val_loss: 0.0085 - val_accuracy: 0.0062\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0055 - val_loss: 0.0085 - val_accuracy: 0.0071\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0058 - val_loss: 0.0086 - val_accuracy: 0.0066\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0063 - val_loss: 0.0084 - val_accuracy: 0.0087\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0065 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0050 - val_loss: 0.0087 - val_accuracy: 0.0077\n",
      "Epoch 11/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0088 - accuracy: 0.0061\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0060 - val_loss: 0.0086 - val_accuracy: 0.0075\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0072 - val_loss: 0.0085 - val_accuracy: 0.0093\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0087 - accuracy: 0.0078 - val_loss: 0.0084 - val_accuracy: 0.0096\n",
      "Epoch 14/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0083\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0083 - val_loss: 0.0084 - val_accuracy: 0.0096\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0087 - val_loss: 0.0084 - val_accuracy: 0.0096\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0072 - val_loss: 0.0084 - val_accuracy: 0.0096\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0086 - accuracy: 0.0084\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0086 - accuracy: 0.0084 - val_loss: 0.0084 - val_accuracy: 0.0098\n",
      "Epoch 18/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0091Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0092 - val_loss: 0.0084 - val_accuracy: 0.0096\n",
      "Epoch 00018: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0627 - accuracy: 0.0476 - val_loss: 0.0230 - val_accuracy: 0.0690\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0238 - accuracy: 0.0836 - val_loss: 0.0222 - val_accuracy: 0.0882\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0229 - accuracy: 0.1030 - val_loss: 0.0213 - val_accuracy: 0.1023\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0222 - accuracy: 0.1150 - val_loss: 0.0212 - val_accuracy: 0.1123\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0215 - accuracy: 0.1241 - val_loss: 0.0209 - val_accuracy: 0.1355\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0212 - accuracy: 0.1309 - val_loss: 0.0210 - val_accuracy: 0.1198\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0209 - accuracy: 0.1346 - val_loss: 0.0206 - val_accuracy: 0.1235\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0205 - accuracy: 0.1425 - val_loss: 0.0204 - val_accuracy: 0.1273\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0201 - accuracy: 0.1521 - val_loss: 0.0205 - val_accuracy: 0.1257\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0197 - accuracy: 0.1605 - val_loss: 0.0204 - val_accuracy: 0.1321\n",
      "Epoch 11/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0193 - accuracy: 0.1730\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0193 - accuracy: 0.1729 - val_loss: 0.0203 - val_accuracy: 0.1358\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0185 - accuracy: 0.1910 - val_loss: 0.0201 - val_accuracy: 0.1390\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0182 - accuracy: 0.1952 - val_loss: 0.0201 - val_accuracy: 0.1426\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0181 - accuracy: 0.1993 - val_loss: 0.0201 - val_accuracy: 0.1401\n",
      "Epoch 15/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0179 - accuracy: 0.2012\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0179 - accuracy: 0.2009 - val_loss: 0.0201 - val_accuracy: 0.1399\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0177 - accuracy: 0.2070 - val_loss: 0.0201 - val_accuracy: 0.1399\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0177 - accuracy: 0.2049 - val_loss: 0.0201 - val_accuracy: 0.1396\n",
      "Epoch 18/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0177 - accuracy: 0.2092\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0177 - accuracy: 0.2099 - val_loss: 0.0201 - val_accuracy: 0.1394\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0177 - accuracy: 0.2077 - val_loss: 0.0201 - val_accuracy: 0.1396\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0177 - accuracy: 0.2055 - val_loss: 0.0201 - val_accuracy: 0.1396\n",
      "Epoch 21/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0177 - accuracy: 0.2079\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0177 - accuracy: 0.2078 - val_loss: 0.0201 - val_accuracy: 0.1399\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0177 - accuracy: 0.2067 - val_loss: 0.0201 - val_accuracy: 0.1396\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0177 - accuracy: 0.2080Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0177 - accuracy: 0.2080 - val_loss: 0.0201 - val_accuracy: 0.1396\n",
      "Epoch 00023: early stopping\n",
      "SEED:30 Fold:1\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2734 - accuracy: 0.0017 - val_loss: 0.0153 - val_accuracy: 0.0032\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0117 - accuracy: 0.0024 - val_loss: 0.0094 - val_accuracy: 0.0050\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0093 - accuracy: 0.0041 - val_loss: 0.0091 - val_accuracy: 0.0052\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0089 - accuracy: 0.0046 - val_loss: 0.0086 - val_accuracy: 0.0048\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0051 - val_loss: 0.0086 - val_accuracy: 0.0055\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.0055 - val_loss: 0.0087 - val_accuracy: 0.0055\n",
      "Epoch 7/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0052\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0087 - accuracy: 0.0051 - val_loss: 0.0086 - val_accuracy: 0.0068\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0086 - accuracy: 0.0060 - val_loss: 0.0086 - val_accuracy: 0.0066\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0086 - accuracy: 0.0079 - val_loss: 0.0086 - val_accuracy: 0.0071\n",
      "Epoch 10/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0085 - accuracy: 0.0085\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0085 - accuracy: 0.0084 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0086 - accuracy: 0.0080 - val_loss: 0.0086 - val_accuracy: 0.0073\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0086 - accuracy: 0.0069 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 13/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0085 - accuracy: 0.0068\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0068 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0083 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0085 - accuracy: 0.0075 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0085 - accuracy: 0.0071\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0071 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0083 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0068 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0085 - accuracy: 0.0075\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0075 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0085 - accuracy: 0.0069Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0069 - val_loss: 0.0085 - val_accuracy: 0.0073\n",
      "Epoch 00020: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0731 - accuracy: 0.0478 - val_loss: 0.0234 - val_accuracy: 0.0736\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0232 - accuracy: 0.0888 - val_loss: 0.0220 - val_accuracy: 0.0957\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0222 - accuracy: 0.1064 - val_loss: 0.0213 - val_accuracy: 0.1048\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0215 - accuracy: 0.1180 - val_loss: 0.0212 - val_accuracy: 0.1139\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0212 - accuracy: 0.1281 - val_loss: 0.0208 - val_accuracy: 0.1125\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0208 - accuracy: 0.1324 - val_loss: 0.0208 - val_accuracy: 0.1219\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0203 - accuracy: 0.1395 - val_loss: 0.0207 - val_accuracy: 0.1260\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0200 - accuracy: 0.1535 - val_loss: 0.0206 - val_accuracy: 0.1237\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0196 - accuracy: 0.1613 - val_loss: 0.0204 - val_accuracy: 0.1321\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0193 - accuracy: 0.1745 - val_loss: 0.0204 - val_accuracy: 0.1321\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0188 - accuracy: 0.1780 - val_loss: 0.0205 - val_accuracy: 0.1346\n",
      "Epoch 12/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0184 - accuracy: 0.1921\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0184 - accuracy: 0.1915 - val_loss: 0.0204 - val_accuracy: 0.1385\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0174 - accuracy: 0.2263 - val_loss: 0.0203 - val_accuracy: 0.1392\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0171 - accuracy: 0.2289 - val_loss: 0.0204 - val_accuracy: 0.1392\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0169 - accuracy: 0.2348 - val_loss: 0.0204 - val_accuracy: 0.1403\n",
      "Epoch 16/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0167 - accuracy: 0.2369\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0167 - accuracy: 0.2370 - val_loss: 0.0205 - val_accuracy: 0.1401\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0165 - accuracy: 0.2423 - val_loss: 0.0205 - val_accuracy: 0.1390\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0165 - accuracy: 0.2439 - val_loss: 0.0205 - val_accuracy: 0.1390\n",
      "Epoch 19/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0165 - accuracy: 0.2468\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0165 - accuracy: 0.2469 - val_loss: 0.0205 - val_accuracy: 0.1385\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0164 - accuracy: 0.2471 - val_loss: 0.0205 - val_accuracy: 0.1387\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0164 - accuracy: 0.2503 - val_loss: 0.0205 - val_accuracy: 0.1390\n",
      "Epoch 22/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0164 - accuracy: 0.2506\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0164 - accuracy: 0.2505 - val_loss: 0.0205 - val_accuracy: 0.1390\n",
      "Epoch 23/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0164 - accuracy: 0.2505Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0165 - accuracy: 0.2496 - val_loss: 0.0205 - val_accuracy: 0.1390\n",
      "Epoch 00023: early stopping\n",
      "SEED:30 Fold:2\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2984 - accuracy: 0.0013 - val_loss: 0.0189 - val_accuracy: 0.0011\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0121 - accuracy: 0.0030 - val_loss: 0.0096 - val_accuracy: 0.0030\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0054 - val_loss: 0.0087 - val_accuracy: 0.0066\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0084 - accuracy: 0.0060 - val_loss: 0.0085 - val_accuracy: 0.0046\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0082 - accuracy: 0.0063 - val_loss: 0.0084 - val_accuracy: 0.0043\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0082 - accuracy: 0.0068 - val_loss: 0.0083 - val_accuracy: 0.0052\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0081 - accuracy: 0.0072 - val_loss: 0.0083 - val_accuracy: 0.0057\n",
      "Epoch 8/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0081 - accuracy: 0.0076\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0081 - accuracy: 0.0075 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0097 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0080 - accuracy: 0.0112 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 11/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0079 - accuracy: 0.0115\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0079 - accuracy: 0.0114 - val_loss: 0.0083 - val_accuracy: 0.0066\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0079 - accuracy: 0.0120 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0079 - accuracy: 0.0126 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0126 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 15/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0079 - accuracy: 0.0117\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0117 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0116 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0131 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0079 - accuracy: 0.0134\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0134 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0124 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0126 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0079 - accuracy: 0.0129\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0129 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0131 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0129 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 24/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0079 - accuracy: 0.0121\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0123 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0079 - accuracy: 0.0130 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 26/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0124 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 27/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0079 - accuracy: 0.0132\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-10.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0132 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 28/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0128 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 29/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0128 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 30/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0079 - accuracy: 0.0127\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-11.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0079 - accuracy: 0.0125 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 00030: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.1133 - accuracy: 0.0531 - val_loss: 0.0247 - val_accuracy: 0.0882\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0226 - accuracy: 0.1202 - val_loss: 0.0214 - val_accuracy: 0.1164\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0207 - accuracy: 0.1445 - val_loss: 0.0205 - val_accuracy: 0.1451\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0198 - accuracy: 0.1543 - val_loss: 0.0202 - val_accuracy: 0.1335\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0191 - accuracy: 0.1754 - val_loss: 0.0200 - val_accuracy: 0.1390\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0184 - accuracy: 0.1939 - val_loss: 0.0199 - val_accuracy: 0.1446\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0176 - accuracy: 0.2166 - val_loss: 0.0201 - val_accuracy: 0.1540\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0167 - accuracy: 0.2456 - val_loss: 0.0202 - val_accuracy: 0.1462\n",
      "Epoch 9/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0159 - accuracy: 0.2729\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0159 - accuracy: 0.2726 - val_loss: 0.0203 - val_accuracy: 0.1508\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0141 - accuracy: 0.3320 - val_loss: 0.0203 - val_accuracy: 0.1519\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0135 - accuracy: 0.3499 - val_loss: 0.0204 - val_accuracy: 0.1565\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0132 - accuracy: 0.3638\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0132 - accuracy: 0.3638 - val_loss: 0.0205 - val_accuracy: 0.1551\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.3765 - val_loss: 0.0205 - val_accuracy: 0.1547\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.3772 - val_loss: 0.0206 - val_accuracy: 0.1547\n",
      "Epoch 15/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0128 - accuracy: 0.3809\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0129 - accuracy: 0.3810 - val_loss: 0.0206 - val_accuracy: 0.1542\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0128 - accuracy: 0.3794Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0128 - accuracy: 0.3794 - val_loss: 0.0206 - val_accuracy: 0.1547\n",
      "Epoch 00016: early stopping\n",
      "SEED:30 Fold:3\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2858 - accuracy: 0.0027 - val_loss: 0.0115 - val_accuracy: 0.0032\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0119 - accuracy: 0.0021 - val_loss: 0.0088 - val_accuracy: 0.0048\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0094 - accuracy: 0.0031 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0089 - accuracy: 0.0047 - val_loss: 0.0085 - val_accuracy: 0.0041\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.0040 - val_loss: 0.0084 - val_accuracy: 0.0048\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0050 - val_loss: 0.0084 - val_accuracy: 0.0048\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0088 - accuracy: 0.0053 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 8/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0088 - accuracy: 0.0057\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0088 - accuracy: 0.0058 - val_loss: 0.0084 - val_accuracy: 0.0052\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0087 - accuracy: 0.0063 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0086 - accuracy: 0.0064 - val_loss: 0.0084 - val_accuracy: 0.0066\n",
      "Epoch 11/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0062\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0086 - accuracy: 0.0062 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0086 - accuracy: 0.0065 - val_loss: 0.0084 - val_accuracy: 0.0082\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0086 - accuracy: 0.0062 - val_loss: 0.0084 - val_accuracy: 0.0077\n",
      "Epoch 14/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0070\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0071 - val_loss: 0.0084 - val_accuracy: 0.0082\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0086 - accuracy: 0.0077 - val_loss: 0.0084 - val_accuracy: 0.0082\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0058 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 17/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0067\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0086 - accuracy: 0.0067 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0086 - accuracy: 0.0062 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0072 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 20/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0075\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0075 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 21/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0067Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0067 - val_loss: 0.0084 - val_accuracy: 0.0080\n",
      "Epoch 00021: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0834 - accuracy: 0.0398 - val_loss: 0.0234 - val_accuracy: 0.0713\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0237 - accuracy: 0.0881 - val_loss: 0.0217 - val_accuracy: 0.0957\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0224 - accuracy: 0.1091 - val_loss: 0.0214 - val_accuracy: 0.1018\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0216 - accuracy: 0.1226 - val_loss: 0.0209 - val_accuracy: 0.1151\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0212 - accuracy: 0.1306 - val_loss: 0.0208 - val_accuracy: 0.1515\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0207 - accuracy: 0.1386 - val_loss: 0.0209 - val_accuracy: 0.1278\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0204 - accuracy: 0.1459 - val_loss: 0.0204 - val_accuracy: 0.1319\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0200 - accuracy: 0.1576 - val_loss: 0.0205 - val_accuracy: 0.1301\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0197 - accuracy: 0.1584 - val_loss: 0.0203 - val_accuracy: 0.1342\n",
      "Epoch 10/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0193 - accuracy: 0.1671\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0193 - accuracy: 0.1670 - val_loss: 0.0203 - val_accuracy: 0.1360\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0186 - accuracy: 0.1851 - val_loss: 0.0201 - val_accuracy: 0.1397\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0183 - accuracy: 0.1904 - val_loss: 0.0201 - val_accuracy: 0.1404\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0182 - accuracy: 0.1923 - val_loss: 0.0201 - val_accuracy: 0.1392\n",
      "Epoch 14/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0181 - accuracy: 0.1984\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0181 - accuracy: 0.1983 - val_loss: 0.0201 - val_accuracy: 0.1406\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0180 - accuracy: 0.1989 - val_loss: 0.0201 - val_accuracy: 0.1394\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0180 - accuracy: 0.1998 - val_loss: 0.0201 - val_accuracy: 0.1404\n",
      "Epoch 17/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0179 - accuracy: 0.1980\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0179 - accuracy: 0.1984 - val_loss: 0.0201 - val_accuracy: 0.1408\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0179 - accuracy: 0.2016 - val_loss: 0.0201 - val_accuracy: 0.1413\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0180 - accuracy: 0.1994 - val_loss: 0.0201 - val_accuracy: 0.1413\n",
      "Epoch 20/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0179 - accuracy: 0.1961\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0179 - accuracy: 0.1965 - val_loss: 0.0201 - val_accuracy: 0.1408\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0179 - accuracy: 0.1986 - val_loss: 0.0201 - val_accuracy: 0.1415\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0179 - accuracy: 0.2004Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 2s 14ms/step - loss: 0.0179 - accuracy: 0.2004 - val_loss: 0.0201 - val_accuracy: 0.1406\n",
      "Epoch 00022: early stopping\n",
      "SEED:30 Fold:4\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2667 - accuracy: 0.0029 - val_loss: 0.0118 - val_accuracy: 0.0021\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0112 - accuracy: 0.0036 - val_loss: 0.0089 - val_accuracy: 0.0050\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0094 - accuracy: 0.0043 - val_loss: 0.0084 - val_accuracy: 0.0041\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0091 - accuracy: 0.0053 - val_loss: 0.0084 - val_accuracy: 0.0043\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0089 - accuracy: 0.0061 - val_loss: 0.0085 - val_accuracy: 0.0048\n",
      "Epoch 6/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0089 - accuracy: 0.0048\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0089 - accuracy: 0.0047 - val_loss: 0.0086 - val_accuracy: 0.0043\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0069 - val_loss: 0.0085 - val_accuracy: 0.0050\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0071 - val_loss: 0.0085 - val_accuracy: 0.0057\n",
      "Epoch 9/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0088 - accuracy: 0.0072\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.0072 - val_loss: 0.0085 - val_accuracy: 0.0055\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0072 - val_loss: 0.0085 - val_accuracy: 0.0057\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0068 - val_loss: 0.0085 - val_accuracy: 0.0057\n",
      "Epoch 12/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0073\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0074 - val_loss: 0.0085 - val_accuracy: 0.0055\n",
      "Epoch 13/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0077Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0077 - val_loss: 0.0085 - val_accuracy: 0.0055\n",
      "Epoch 00013: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0720 - accuracy: 0.0555 - val_loss: 0.0230 - val_accuracy: 0.0886\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0231 - accuracy: 0.1118 - val_loss: 0.0217 - val_accuracy: 0.1059\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0217 - accuracy: 0.1295 - val_loss: 0.0210 - val_accuracy: 0.1233\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0206 - accuracy: 0.1481 - val_loss: 0.0207 - val_accuracy: 0.1280\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0198 - accuracy: 0.1696 - val_loss: 0.0206 - val_accuracy: 0.1374\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0188 - accuracy: 0.1918 - val_loss: 0.0206 - val_accuracy: 0.1497\n",
      "Epoch 7/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0178 - accuracy: 0.2273\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0178 - accuracy: 0.2272 - val_loss: 0.0208 - val_accuracy: 0.1415\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0156 - accuracy: 0.2990 - val_loss: 0.0203 - val_accuracy: 0.1504\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0148 - accuracy: 0.3270 - val_loss: 0.0203 - val_accuracy: 0.1497\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0141 - accuracy: 0.3541 - val_loss: 0.0204 - val_accuracy: 0.1467\n",
      "Epoch 11/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0135 - accuracy: 0.3698\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 2s 14ms/step - loss: 0.0135 - accuracy: 0.3692 - val_loss: 0.0206 - val_accuracy: 0.1447\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0130 - accuracy: 0.3868 - val_loss: 0.0207 - val_accuracy: 0.1465\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0129 - accuracy: 0.3915 - val_loss: 0.0207 - val_accuracy: 0.1467\n",
      "Epoch 14/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0128 - accuracy: 0.3965\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0128 - accuracy: 0.3955 - val_loss: 0.0207 - val_accuracy: 0.1476\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0128 - accuracy: 0.3995 - val_loss: 0.0208 - val_accuracy: 0.1483\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0128 - accuracy: 0.3956 - val_loss: 0.0208 - val_accuracy: 0.1479\n",
      "Epoch 17/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0128 - accuracy: 0.3981\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0128 - accuracy: 0.3970 - val_loss: 0.0208 - val_accuracy: 0.1479\n",
      "Epoch 18/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0128 - accuracy: 0.3976Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0128 - accuracy: 0.3973 - val_loss: 0.0208 - val_accuracy: 0.1474\n",
      "Epoch 00018: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=58 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:58 Fold:0\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.2647 - accuracy: 0.0030 - val_loss: 0.0137 - val_accuracy: 0.0018\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0110 - accuracy: 0.0036 - val_loss: 0.0089 - val_accuracy: 0.0048\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0091 - accuracy: 0.0048 - val_loss: 0.0084 - val_accuracy: 0.0046\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0091 - accuracy: 0.0052 - val_loss: 0.0085 - val_accuracy: 0.0048\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0088 - accuracy: 0.0054 - val_loss: 0.0085 - val_accuracy: 0.0043\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0087 - accuracy: 0.0063\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0087 - accuracy: 0.0063 - val_loss: 0.0085 - val_accuracy: 0.0066\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0081 - val_loss: 0.0085 - val_accuracy: 0.0066\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0087 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 9/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0095\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0095 - val_loss: 0.0085 - val_accuracy: 0.0057\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0085 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0095 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 12/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0085 - accuracy: 0.0094\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0085 - accuracy: 0.0093 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0099 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0080 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 15/150\n",
      "128/138 [==========================>...] - ETA: 0s - loss: 0.0085 - accuracy: 0.0096\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0097 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0095 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0085 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 18/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0085 - accuracy: 0.0104\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0103 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0091 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0104 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 21/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0085 - accuracy: 0.0089\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0088 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0087 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0085 - accuracy: 0.0096 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 24/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0085 - accuracy: 0.0086\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-10.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0090 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0085 - accuracy: 0.0089 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 26/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0090 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 27/150\n",
      "128/138 [==========================>...] - ETA: 0s - loss: 0.0085 - accuracy: 0.0101\n",
      "Epoch 00027: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-11.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0100 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 28/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0093 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 29/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0088 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 30/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0087\n",
      "Epoch 00030: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-12.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0085 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 31/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0091 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 32/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0088 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 33/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0085 - accuracy: 0.0098\n",
      "Epoch 00033: ReduceLROnPlateau reducing learning rate to 1.0000001044244145e-13.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0098 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 00033: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0748 - accuracy: 0.0608 - val_loss: 0.0228 - val_accuracy: 0.0954\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0224 - accuracy: 0.1167 - val_loss: 0.0212 - val_accuracy: 0.1424\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0210 - accuracy: 0.1421 - val_loss: 0.0209 - val_accuracy: 0.1241\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0201 - accuracy: 0.1668 - val_loss: 0.0206 - val_accuracy: 0.1355\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0193 - accuracy: 0.1792 - val_loss: 0.0204 - val_accuracy: 0.1581\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0184 - accuracy: 0.2032 - val_loss: 0.0203 - val_accuracy: 0.1442\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0173 - accuracy: 0.2294 - val_loss: 0.0204 - val_accuracy: 0.1478\n",
      "Epoch 8/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0165 - accuracy: 0.2573\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0166 - accuracy: 0.2574 - val_loss: 0.0205 - val_accuracy: 0.1467\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0147 - accuracy: 0.3169 - val_loss: 0.0203 - val_accuracy: 0.1476\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0143 - accuracy: 0.3376 - val_loss: 0.0204 - val_accuracy: 0.1497\n",
      "Epoch 11/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0140 - accuracy: 0.3449\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0140 - accuracy: 0.3445 - val_loss: 0.0204 - val_accuracy: 0.1508\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0137 - accuracy: 0.3558 - val_loss: 0.0205 - val_accuracy: 0.1510\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0136 - accuracy: 0.3584 - val_loss: 0.0205 - val_accuracy: 0.1524\n",
      "Epoch 14/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0136 - accuracy: 0.3591\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0136 - accuracy: 0.3591 - val_loss: 0.0205 - val_accuracy: 0.1522\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0136 - accuracy: 0.3607 - val_loss: 0.0205 - val_accuracy: 0.1522\n",
      "Epoch 16/150\n",
      "129/138 [===========================>..] - ETA: 0s - loss: 0.0135 - accuracy: 0.3623Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0135 - accuracy: 0.3629 - val_loss: 0.0205 - val_accuracy: 0.1522\n",
      "Epoch 00016: early stopping\n",
      "SEED:58 Fold:1\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.2787 - accuracy: 0.0026 - val_loss: 0.0132 - val_accuracy: 0.0025\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0119 - accuracy: 0.0032 - val_loss: 0.0091 - val_accuracy: 0.0043\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0094 - accuracy: 0.0053 - val_loss: 0.0087 - val_accuracy: 0.0030\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0091 - accuracy: 0.0062 - val_loss: 0.0085 - val_accuracy: 0.0036\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0090 - accuracy: 0.0060 - val_loss: 0.0085 - val_accuracy: 0.0036\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0088 - accuracy: 0.0062 - val_loss: 0.0084 - val_accuracy: 0.0036\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0088 - accuracy: 0.0075 - val_loss: 0.0084 - val_accuracy: 0.0046\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0077 - val_loss: 0.0084 - val_accuracy: 0.0055\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.0073 - val_loss: 0.0085 - val_accuracy: 0.0043\n",
      "Epoch 10/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0088 - accuracy: 0.0084\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.0086 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0097 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0096 - val_loss: 0.0084 - val_accuracy: 0.0059\n",
      "Epoch 13/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0096\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0086 - accuracy: 0.0096 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0099 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0104 - val_loss: 0.0083 - val_accuracy: 0.0064\n",
      "Epoch 16/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0091\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0095 - val_loss: 0.0084 - val_accuracy: 0.0062\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0099 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0087 - accuracy: 0.0102 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 19/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0087 - accuracy: 0.0092\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0087 - accuracy: 0.0098 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0087 - accuracy: 0.0093 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0095 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 22/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0087 - accuracy: 0.0099\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0087 - accuracy: 0.0099 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0096 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0087 - accuracy: 0.0098 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 25/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0095\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0096 - val_loss: 0.0084 - val_accuracy: 0.0066\n",
      "Epoch 00025: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0722 - accuracy: 0.0452 - val_loss: 0.0241 - val_accuracy: 0.0731\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0237 - accuracy: 0.0933 - val_loss: 0.0225 - val_accuracy: 0.1246\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0226 - accuracy: 0.1144 - val_loss: 0.0217 - val_accuracy: 0.1062\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0219 - accuracy: 0.1226 - val_loss: 0.0213 - val_accuracy: 0.1119\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0214 - accuracy: 0.1319 - val_loss: 0.0211 - val_accuracy: 0.1187\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0209 - accuracy: 0.1406 - val_loss: 0.0208 - val_accuracy: 0.1210\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0204 - accuracy: 0.1477 - val_loss: 0.0205 - val_accuracy: 0.1335\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0201 - accuracy: 0.1525 - val_loss: 0.0203 - val_accuracy: 0.1326\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0195 - accuracy: 0.1649 - val_loss: 0.0203 - val_accuracy: 0.1376\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0191 - accuracy: 0.1783 - val_loss: 0.0201 - val_accuracy: 0.1404\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0187 - accuracy: 0.1883 - val_loss: 0.0200 - val_accuracy: 0.1447\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0183 - accuracy: 0.1970 - val_loss: 0.0201 - val_accuracy: 0.1472\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0180 - accuracy: 0.2033 - val_loss: 0.0199 - val_accuracy: 0.1472\n",
      "Epoch 14/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0176 - accuracy: 0.2181\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0176 - accuracy: 0.2181 - val_loss: 0.0201 - val_accuracy: 0.1472\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0168 - accuracy: 0.2410 - val_loss: 0.0199 - val_accuracy: 0.1515\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0166 - accuracy: 0.2449 - val_loss: 0.0199 - val_accuracy: 0.1538\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0164 - accuracy: 0.2533 - val_loss: 0.0199 - val_accuracy: 0.1547\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0162 - accuracy: 0.2599 - val_loss: 0.0199 - val_accuracy: 0.1561\n",
      "Epoch 19/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0161 - accuracy: 0.2604\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0161 - accuracy: 0.2603 - val_loss: 0.0199 - val_accuracy: 0.1542\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0160 - accuracy: 0.2629 - val_loss: 0.0199 - val_accuracy: 0.1542\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0159 - accuracy: 0.2706 - val_loss: 0.0199 - val_accuracy: 0.1529\n",
      "Epoch 22/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0159 - accuracy: 0.2687\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0159 - accuracy: 0.2686 - val_loss: 0.0199 - val_accuracy: 0.1529\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0159 - accuracy: 0.2668 - val_loss: 0.0199 - val_accuracy: 0.1522\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0159 - accuracy: 0.2669 - val_loss: 0.0199 - val_accuracy: 0.1527\n",
      "Epoch 25/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0159 - accuracy: 0.2673\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0159 - accuracy: 0.2673 - val_loss: 0.0199 - val_accuracy: 0.1529\n",
      "Epoch 26/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0160 - accuracy: 0.2671Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0160 - accuracy: 0.2669 - val_loss: 0.0199 - val_accuracy: 0.1529\n",
      "Epoch 00026: early stopping\n",
      "SEED:58 Fold:2\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.2705 - accuracy: 0.0018 - val_loss: 0.0140 - val_accuracy: 0.0016\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0110 - accuracy: 0.0032 - val_loss: 0.0090 - val_accuracy: 0.0082\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0088 - accuracy: 0.0052 - val_loss: 0.0085 - val_accuracy: 0.0050\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0084 - accuracy: 0.0048 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0084 - accuracy: 0.0056 - val_loss: 0.0084 - val_accuracy: 0.0052\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0083 - accuracy: 0.0065 - val_loss: 0.0084 - val_accuracy: 0.0068\n",
      "Epoch 7/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0083 - accuracy: 0.0072\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0083 - accuracy: 0.0071 - val_loss: 0.0083 - val_accuracy: 0.0055\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0082 - accuracy: 0.0087 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0093 - val_loss: 0.0083 - val_accuracy: 0.0075\n",
      "Epoch 10/150\n",
      "129/138 [===========================>..] - ETA: 0s - loss: 0.0081 - accuracy: 0.0094\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0095 - val_loss: 0.0083 - val_accuracy: 0.0075\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0106 - val_loss: 0.0083 - val_accuracy: 0.0084\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0081 - accuracy: 0.0114 - val_loss: 0.0083 - val_accuracy: 0.0089\n",
      "Epoch 13/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0081 - accuracy: 0.0116\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0081 - accuracy: 0.0119 - val_loss: 0.0083 - val_accuracy: 0.0087\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0105 - val_loss: 0.0083 - val_accuracy: 0.0087\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0081 - accuracy: 0.0111 - val_loss: 0.0083 - val_accuracy: 0.0087\n",
      "Epoch 16/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0081 - accuracy: 0.0109\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0107 - val_loss: 0.0083 - val_accuracy: 0.0087\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0121 - val_loss: 0.0083 - val_accuracy: 0.0082\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0106 - val_loss: 0.0083 - val_accuracy: 0.0087\n",
      "Epoch 19/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0081 - accuracy: 0.0107\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0107 - val_loss: 0.0083 - val_accuracy: 0.0084\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0106 - val_loss: 0.0083 - val_accuracy: 0.0087\n",
      "Epoch 21/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0081 - accuracy: 0.0106Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0109 - val_loss: 0.0083 - val_accuracy: 0.0084\n",
      "Epoch 00021: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0957 - accuracy: 0.0543 - val_loss: 0.0233 - val_accuracy: 0.1018\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0224 - accuracy: 0.1183 - val_loss: 0.0211 - val_accuracy: 0.1381\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0209 - accuracy: 0.1441 - val_loss: 0.0204 - val_accuracy: 0.1429\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0199 - accuracy: 0.1619 - val_loss: 0.0201 - val_accuracy: 0.1479\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0190 - accuracy: 0.1794 - val_loss: 0.0200 - val_accuracy: 0.1634\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0184 - accuracy: 0.1974 - val_loss: 0.0200 - val_accuracy: 0.1531\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0175 - accuracy: 0.2180 - val_loss: 0.0199 - val_accuracy: 0.1527\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0168 - accuracy: 0.2378 - val_loss: 0.0201 - val_accuracy: 0.1540\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0162 - accuracy: 0.2602 - val_loss: 0.0202 - val_accuracy: 0.1568\n",
      "Epoch 10/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0155 - accuracy: 0.2824\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0155 - accuracy: 0.2824 - val_loss: 0.0203 - val_accuracy: 0.1572\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0142 - accuracy: 0.3306 - val_loss: 0.0201 - val_accuracy: 0.1638\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0139 - accuracy: 0.3356 - val_loss: 0.0201 - val_accuracy: 0.1631\n",
      "Epoch 13/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0136 - accuracy: 0.3478\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0136 - accuracy: 0.3477 - val_loss: 0.0202 - val_accuracy: 0.1661\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0134 - accuracy: 0.3587 - val_loss: 0.0202 - val_accuracy: 0.1654\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0133 - accuracy: 0.3599 - val_loss: 0.0202 - val_accuracy: 0.1647\n",
      "Epoch 16/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0133 - accuracy: 0.3570\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0133 - accuracy: 0.3570 - val_loss: 0.0202 - val_accuracy: 0.1640\n",
      "Epoch 17/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0133 - accuracy: 0.3598Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0133 - accuracy: 0.3604 - val_loss: 0.0202 - val_accuracy: 0.1645\n",
      "Epoch 00017: early stopping\n",
      "SEED:58 Fold:3\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.2935 - accuracy: 0.0013 - val_loss: 0.0146 - val_accuracy: 0.0027\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0120 - accuracy: 0.0040 - val_loss: 0.0093 - val_accuracy: 0.0046\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0089 - accuracy: 0.0060 - val_loss: 0.0086 - val_accuracy: 0.0048\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0084 - accuracy: 0.0061 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0082 - accuracy: 0.0062 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0082 - accuracy: 0.0070 - val_loss: 0.0083 - val_accuracy: 0.0055\n",
      "Epoch 7/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0081 - accuracy: 0.0069\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0081 - accuracy: 0.0069 - val_loss: 0.0083 - val_accuracy: 0.0057\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0081 - accuracy: 0.0087 - val_loss: 0.0083 - val_accuracy: 0.0071\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0089 - val_loss: 0.0083 - val_accuracy: 0.0068\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0103 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 11/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0080 - accuracy: 0.0098\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0080 - accuracy: 0.0097 - val_loss: 0.0083 - val_accuracy: 0.0075\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0080 - accuracy: 0.0100 - val_loss: 0.0083 - val_accuracy: 0.0075\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0104 - val_loss: 0.0083 - val_accuracy: 0.0080\n",
      "Epoch 14/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0080 - accuracy: 0.0106\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0080 - accuracy: 0.0104 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0080 - accuracy: 0.0103 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0080 - accuracy: 0.0110 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 17/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0080 - accuracy: 0.0106\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0080 - accuracy: 0.0107 - val_loss: 0.0083 - val_accuracy: 0.0077\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0104 - val_loss: 0.0083 - val_accuracy: 0.0075\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0080 - accuracy: 0.0107 - val_loss: 0.0083 - val_accuracy: 0.0077\n",
      "Epoch 20/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0080 - accuracy: 0.0100\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0101 - val_loss: 0.0083 - val_accuracy: 0.0080\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0113 - val_loss: 0.0083 - val_accuracy: 0.0077\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0107 - val_loss: 0.0083 - val_accuracy: 0.0073\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0080 - accuracy: 0.0109\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0080 - accuracy: 0.0109 - val_loss: 0.0083 - val_accuracy: 0.0075\n",
      "Epoch 24/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0080 - accuracy: 0.0104Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0080 - accuracy: 0.0104 - val_loss: 0.0083 - val_accuracy: 0.0075\n",
      "Epoch 00024: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.1203 - accuracy: 0.0421 - val_loss: 0.0250 - val_accuracy: 0.0952\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0228 - accuracy: 0.1178 - val_loss: 0.0219 - val_accuracy: 0.1410\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0207 - accuracy: 0.1359 - val_loss: 0.0209 - val_accuracy: 0.1542\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0199 - accuracy: 0.1531 - val_loss: 0.0203 - val_accuracy: 0.1308\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0192 - accuracy: 0.1691 - val_loss: 0.0200 - val_accuracy: 0.1403\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0185 - accuracy: 0.1850 - val_loss: 0.0200 - val_accuracy: 0.1458\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0179 - accuracy: 0.2050 - val_loss: 0.0200 - val_accuracy: 0.1426\n",
      "Epoch 8/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0172 - accuracy: 0.2242\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0172 - accuracy: 0.2245 - val_loss: 0.0199 - val_accuracy: 0.1449\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0158 - accuracy: 0.2615 - val_loss: 0.0198 - val_accuracy: 0.1501\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0155 - accuracy: 0.2739 - val_loss: 0.0198 - val_accuracy: 0.1487\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0153 - accuracy: 0.2808 - val_loss: 0.0199 - val_accuracy: 0.1508\n",
      "Epoch 12/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0151 - accuracy: 0.2863\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0151 - accuracy: 0.2860 - val_loss: 0.0199 - val_accuracy: 0.1506\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0148 - accuracy: 0.2919 - val_loss: 0.0199 - val_accuracy: 0.1510\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0148 - accuracy: 0.2947 - val_loss: 0.0200 - val_accuracy: 0.1524\n",
      "Epoch 15/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0148 - accuracy: 0.2976\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0148 - accuracy: 0.2978 - val_loss: 0.0200 - val_accuracy: 0.1513\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0148 - accuracy: 0.2952 - val_loss: 0.0200 - val_accuracy: 0.1522\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0148 - accuracy: 0.2934 - val_loss: 0.0200 - val_accuracy: 0.1522\n",
      "Epoch 18/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0147 - accuracy: 0.2982\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0147 - accuracy: 0.2992 - val_loss: 0.0200 - val_accuracy: 0.1513\n",
      "Epoch 19/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0148 - accuracy: 0.2942Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0148 - accuracy: 0.2942 - val_loss: 0.0200 - val_accuracy: 0.1513\n",
      "Epoch 00019: early stopping\n",
      "SEED:58 Fold:4\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.2685 - accuracy: 0.0018 - val_loss: 0.0117 - val_accuracy: 0.0055\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0113 - accuracy: 0.0026 - val_loss: 0.0088 - val_accuracy: 0.0068\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0093 - accuracy: 0.0027 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0092 - accuracy: 0.0026 - val_loss: 0.0086 - val_accuracy: 0.0062\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0090 - accuracy: 0.0046 - val_loss: 0.0084 - val_accuracy: 0.0062\n",
      "Epoch 6/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0090 - accuracy: 0.0047\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0090 - accuracy: 0.0047 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0052 - val_loss: 0.0084 - val_accuracy: 0.0059\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0089 - accuracy: 0.0049 - val_loss: 0.0084 - val_accuracy: 0.0062\n",
      "Epoch 9/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0088 - accuracy: 0.0048\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0088 - accuracy: 0.0048 - val_loss: 0.0084 - val_accuracy: 0.0066\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0088 - accuracy: 0.0058 - val_loss: 0.0084 - val_accuracy: 0.0059\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0089 - accuracy: 0.0063 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 12/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0089 - accuracy: 0.0056\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0089 - accuracy: 0.0056 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0089 - accuracy: 0.0055 - val_loss: 0.0084 - val_accuracy: 0.0059\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0089 - accuracy: 0.0052 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 15/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0089 - accuracy: 0.0046\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0089 - accuracy: 0.0046 - val_loss: 0.0084 - val_accuracy: 0.0062\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0088 - accuracy: 0.0046 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0088 - accuracy: 0.0051 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 18/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0089 - accuracy: 0.0051\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0054 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 19/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0089 - accuracy: 0.0055Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0055 - val_loss: 0.0084 - val_accuracy: 0.0064\n",
      "Epoch 00019: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0746 - accuracy: 0.0358 - val_loss: 0.0236 - val_accuracy: 0.0706\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0243 - accuracy: 0.0767 - val_loss: 0.0221 - val_accuracy: 0.0870\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0232 - accuracy: 0.0992 - val_loss: 0.0216 - val_accuracy: 0.1091\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0225 - accuracy: 0.1117 - val_loss: 0.0215 - val_accuracy: 0.1075\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0220 - accuracy: 0.1139 - val_loss: 0.0214 - val_accuracy: 0.1098\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0216 - accuracy: 0.1213 - val_loss: 0.0210 - val_accuracy: 0.1219\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0213 - accuracy: 0.1256 - val_loss: 0.0208 - val_accuracy: 0.1203\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0209 - accuracy: 0.1328 - val_loss: 0.0208 - val_accuracy: 0.1239\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0207 - accuracy: 0.1334 - val_loss: 0.0206 - val_accuracy: 0.1276\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0204 - accuracy: 0.1394 - val_loss: 0.0208 - val_accuracy: 0.1303\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0203 - accuracy: 0.1420 - val_loss: 0.0205 - val_accuracy: 0.1278\n",
      "Epoch 12/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0201 - accuracy: 0.1448\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0201 - accuracy: 0.1448 - val_loss: 0.0206 - val_accuracy: 0.1317\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0196 - accuracy: 0.1509 - val_loss: 0.0204 - val_accuracy: 0.1323\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0194 - accuracy: 0.1577 - val_loss: 0.0203 - val_accuracy: 0.1358\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0194 - accuracy: 0.1580 - val_loss: 0.0203 - val_accuracy: 0.1355\n",
      "Epoch 16/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0193 - accuracy: 0.1625\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0193 - accuracy: 0.1623 - val_loss: 0.0203 - val_accuracy: 0.1330\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1615 - val_loss: 0.0203 - val_accuracy: 0.1335\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1626 - val_loss: 0.0203 - val_accuracy: 0.1342\n",
      "Epoch 19/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0192 - accuracy: 0.1603\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1600 - val_loss: 0.0203 - val_accuracy: 0.1346\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1624 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1645 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0192 - accuracy: 0.1590 - val_loss: 0.0203 - val_accuracy: 0.1346\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1609 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1625 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 25/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0192 - accuracy: 0.1604\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1593 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 26/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1615 - val_loss: 0.0203 - val_accuracy: 0.1355\n",
      "Epoch 27/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0192 - accuracy: 0.1599 - val_loss: 0.0203 - val_accuracy: 0.1349\n",
      "Epoch 28/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0192 - accuracy: 0.1621\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0192 - accuracy: 0.1624 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 29/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1619 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 30/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1617 - val_loss: 0.0203 - val_accuracy: 0.1346\n",
      "Epoch 31/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0191 - accuracy: 0.1634\n",
      "Epoch 00031: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0192 - accuracy: 0.1636 - val_loss: 0.0203 - val_accuracy: 0.1344\n",
      "Epoch 32/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0192 - accuracy: 0.1621Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0192 - accuracy: 0.1618 - val_loss: 0.0203 - val_accuracy: 0.1349\n",
      "Epoch 00032: early stopping\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=3255 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:3255 Fold:0\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2903 - accuracy: 0.0107 - val_loss: 0.0168 - val_accuracy: 0.0018\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0119 - accuracy: 0.0034 - val_loss: 0.0096 - val_accuracy: 0.0052\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0089 - accuracy: 0.0038 - val_loss: 0.0085 - val_accuracy: 0.0050\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0084 - accuracy: 0.0044 - val_loss: 0.0083 - val_accuracy: 0.0046\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0083 - accuracy: 0.0059 - val_loss: 0.0082 - val_accuracy: 0.0050\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0082 - accuracy: 0.0058 - val_loss: 0.0082 - val_accuracy: 0.0055\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0082 - accuracy: 0.0060 - val_loss: 0.0082 - val_accuracy: 0.0077\n",
      "Epoch 8/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0082 - accuracy: 0.0061\n",
      "Epoch 00008: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0082 - accuracy: 0.0062 - val_loss: 0.0081 - val_accuracy: 0.0077\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0080 - accuracy: 0.0101 - val_loss: 0.0081 - val_accuracy: 0.0096\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0080 - accuracy: 0.0101 - val_loss: 0.0081 - val_accuracy: 0.0096\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0080 - accuracy: 0.0104 - val_loss: 0.0081 - val_accuracy: 0.0098\n",
      "Epoch 12/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0080 - accuracy: 0.0112\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0080 - accuracy: 0.0111 - val_loss: 0.0081 - val_accuracy: 0.0089\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 2s 16ms/step - loss: 0.0079 - accuracy: 0.0113 - val_loss: 0.0081 - val_accuracy: 0.0093\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0079 - accuracy: 0.0124 - val_loss: 0.0081 - val_accuracy: 0.0103\n",
      "Epoch 15/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0079 - accuracy: 0.0125\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0079 - accuracy: 0.0127 - val_loss: 0.0080 - val_accuracy: 0.0107\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 2s 14ms/step - loss: 0.0079 - accuracy: 0.0126 - val_loss: 0.0081 - val_accuracy: 0.0107\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 2s 14ms/step - loss: 0.0079 - accuracy: 0.0128 - val_loss: 0.0081 - val_accuracy: 0.0107\n",
      "Epoch 18/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0079 - accuracy: 0.0120\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0079 - accuracy: 0.0120 - val_loss: 0.0081 - val_accuracy: 0.0109\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0079 - accuracy: 0.0129 - val_loss: 0.0081 - val_accuracy: 0.0107\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0079 - accuracy: 0.0128 - val_loss: 0.0080 - val_accuracy: 0.0107\n",
      "Epoch 21/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0079 - accuracy: 0.0129\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0079 - accuracy: 0.0127 - val_loss: 0.0081 - val_accuracy: 0.0105\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0079 - accuracy: 0.0124 - val_loss: 0.0081 - val_accuracy: 0.0107\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0079 - accuracy: 0.0122 - val_loss: 0.0081 - val_accuracy: 0.0109\n",
      "Epoch 24/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0079 - accuracy: 0.0129\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0079 - accuracy: 0.0129 - val_loss: 0.0080 - val_accuracy: 0.0107\n",
      "Epoch 25/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0079 - accuracy: 0.0130Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0079 - accuracy: 0.0130 - val_loss: 0.0081 - val_accuracy: 0.0107\n",
      "Epoch 00025: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.1232 - accuracy: 0.0441 - val_loss: 0.0265 - val_accuracy: 0.0610\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0231 - accuracy: 0.1002 - val_loss: 0.0217 - val_accuracy: 0.1082\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0210 - accuracy: 0.1241 - val_loss: 0.0207 - val_accuracy: 0.1467\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0201 - accuracy: 0.1416 - val_loss: 0.0203 - val_accuracy: 0.1317\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0195 - accuracy: 0.1543 - val_loss: 0.0204 - val_accuracy: 0.1278\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0189 - accuracy: 0.1714 - val_loss: 0.0202 - val_accuracy: 0.1394\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0184 - accuracy: 0.1888 - val_loss: 0.0203 - val_accuracy: 0.1367\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0178 - accuracy: 0.2015 - val_loss: 0.0201 - val_accuracy: 0.1526\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0173 - accuracy: 0.2188 - val_loss: 0.0202 - val_accuracy: 0.1476\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0166 - accuracy: 0.2456 - val_loss: 0.0205 - val_accuracy: 0.1435\n",
      "Epoch 11/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0161 - accuracy: 0.2527\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0161 - accuracy: 0.2500 - val_loss: 0.0207 - val_accuracy: 0.1451\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0147 - accuracy: 0.2953 - val_loss: 0.0204 - val_accuracy: 0.1490\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0143 - accuracy: 0.3112 - val_loss: 0.0205 - val_accuracy: 0.1485\n",
      "Epoch 14/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0141 - accuracy: 0.3227\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0141 - accuracy: 0.3225 - val_loss: 0.0206 - val_accuracy: 0.1487\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0138 - accuracy: 0.3255 - val_loss: 0.0207 - val_accuracy: 0.1492\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0138 - accuracy: 0.3297 - val_loss: 0.0207 - val_accuracy: 0.1487\n",
      "Epoch 17/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0138 - accuracy: 0.3289\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0138 - accuracy: 0.3285 - val_loss: 0.0207 - val_accuracy: 0.1490\n",
      "Epoch 18/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0137 - accuracy: 0.3283Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0137 - accuracy: 0.3286 - val_loss: 0.0207 - val_accuracy: 0.1494\n",
      "Epoch 00018: early stopping\n",
      "SEED:3255 Fold:1\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.2718 - accuracy: 0.0017 - val_loss: 0.0120 - val_accuracy: 0.0025\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0114 - accuracy: 0.0026 - val_loss: 0.0088 - val_accuracy: 0.0043\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0094 - accuracy: 0.0036 - val_loss: 0.0085 - val_accuracy: 0.0043\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0090 - accuracy: 0.0038 - val_loss: 0.0085 - val_accuracy: 0.0050\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0090 - accuracy: 0.0048 - val_loss: 0.0084 - val_accuracy: 0.0041\n",
      "Epoch 6/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0090 - accuracy: 0.0045\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0090 - accuracy: 0.0045 - val_loss: 0.0085 - val_accuracy: 0.0055\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0089 - accuracy: 0.0058 - val_loss: 0.0085 - val_accuracy: 0.0055\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0088 - accuracy: 0.0055 - val_loss: 0.0085 - val_accuracy: 0.0057\n",
      "Epoch 9/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0089 - accuracy: 0.0062\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0089 - accuracy: 0.0062 - val_loss: 0.0085 - val_accuracy: 0.0052\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0089 - accuracy: 0.0058 - val_loss: 0.0085 - val_accuracy: 0.0057\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0089 - accuracy: 0.0070 - val_loss: 0.0085 - val_accuracy: 0.0057\n",
      "Epoch 12/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0088 - accuracy: 0.0058\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0088 - accuracy: 0.0058 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0089 - accuracy: 0.0062 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0088 - accuracy: 0.0055 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 15/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0088 - accuracy: 0.0060\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0088 - accuracy: 0.0059 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 00015: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0739 - accuracy: 0.0420 - val_loss: 0.0233 - val_accuracy: 0.1132\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0241 - accuracy: 0.0878 - val_loss: 0.0218 - val_accuracy: 0.1066\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0228 - accuracy: 0.1093 - val_loss: 0.0212 - val_accuracy: 0.1110\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0221 - accuracy: 0.1199 - val_loss: 0.0209 - val_accuracy: 0.1167\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0216 - accuracy: 0.1286 - val_loss: 0.0208 - val_accuracy: 0.1260\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0211 - accuracy: 0.1358 - val_loss: 0.0205 - val_accuracy: 0.1326\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0208 - accuracy: 0.1419 - val_loss: 0.0204 - val_accuracy: 0.1285\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0205 - accuracy: 0.1486 - val_loss: 0.0205 - val_accuracy: 0.1324\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0202 - accuracy: 0.1548 - val_loss: 0.0202 - val_accuracy: 0.1333\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0198 - accuracy: 0.1584 - val_loss: 0.0202 - val_accuracy: 0.1376\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0195 - accuracy: 0.1687 - val_loss: 0.0200 - val_accuracy: 0.1419\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0193 - accuracy: 0.1749 - val_loss: 0.0201 - val_accuracy: 0.1438\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0190 - accuracy: 0.1767 - val_loss: 0.0199 - val_accuracy: 0.1442\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0187 - accuracy: 0.1861 - val_loss: 0.0200 - val_accuracy: 0.1445\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0185 - accuracy: 0.1940 - val_loss: 0.0200 - val_accuracy: 0.1442\n",
      "Epoch 16/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0182 - accuracy: 0.2021\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0182 - accuracy: 0.2022 - val_loss: 0.0201 - val_accuracy: 0.1486\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0175 - accuracy: 0.2221 - val_loss: 0.0199 - val_accuracy: 0.1474\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0174 - accuracy: 0.2237 - val_loss: 0.0199 - val_accuracy: 0.1492\n",
      "Epoch 19/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0172 - accuracy: 0.2304\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0172 - accuracy: 0.2299 - val_loss: 0.0199 - val_accuracy: 0.1497\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0172 - accuracy: 0.2233 - val_loss: 0.0199 - val_accuracy: 0.1504\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0171 - accuracy: 0.2293 - val_loss: 0.0199 - val_accuracy: 0.1501\n",
      "Epoch 22/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0171 - accuracy: 0.2305\n",
      "Epoch 00022: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0171 - accuracy: 0.2308 - val_loss: 0.0199 - val_accuracy: 0.1497\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0171 - accuracy: 0.2273 - val_loss: 0.0199 - val_accuracy: 0.1492\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0170 - accuracy: 0.2324 - val_loss: 0.0199 - val_accuracy: 0.1492\n",
      "Epoch 25/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0171 - accuracy: 0.2289\n",
      "Epoch 00025: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0171 - accuracy: 0.2288 - val_loss: 0.0199 - val_accuracy: 0.1490\n",
      "Epoch 26/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0170 - accuracy: 0.2328 - val_loss: 0.0199 - val_accuracy: 0.1495\n",
      "Epoch 27/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0170 - accuracy: 0.2324 - val_loss: 0.0199 - val_accuracy: 0.1501\n",
      "Epoch 28/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0171 - accuracy: 0.2323\n",
      "Epoch 00028: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0171 - accuracy: 0.2328 - val_loss: 0.0199 - val_accuracy: 0.1504\n",
      "Epoch 29/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0171 - accuracy: 0.2296Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0171 - accuracy: 0.2295 - val_loss: 0.0199 - val_accuracy: 0.1495\n",
      "Epoch 00029: early stopping\n",
      "SEED:3255 Fold:2\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.2619 - accuracy: 0.0023 - val_loss: 0.0117 - val_accuracy: 9.1116e-04\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0109 - accuracy: 0.0026 - val_loss: 0.0090 - val_accuracy: 0.0048\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0092 - accuracy: 0.0031 - val_loss: 0.0087 - val_accuracy: 0.0032\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0090 - accuracy: 0.0038 - val_loss: 0.0087 - val_accuracy: 0.0055\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0089 - accuracy: 0.0048 - val_loss: 0.0087 - val_accuracy: 0.0039\n",
      "Epoch 6/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0088 - accuracy: 0.0046\n",
      "Epoch 00006: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0048 - val_loss: 0.0087 - val_accuracy: 0.0077\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0050 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0088 - accuracy: 0.0054 - val_loss: 0.0087 - val_accuracy: 0.0062\n",
      "Epoch 9/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0056\n",
      "Epoch 00009: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0087 - accuracy: 0.0055 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0066 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0060 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 12/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0059\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0059 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0087 - accuracy: 0.0061 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0087 - accuracy: 0.0058 - val_loss: 0.0087 - val_accuracy: 0.0059\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0087 - accuracy: 0.0060\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0060 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0064 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0062 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 18/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0064\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0087 - accuracy: 0.0064 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0087 - accuracy: 0.0060 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0062 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 21/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0057\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-09.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0056 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0087 - accuracy: 0.0063 - val_loss: 0.0087 - val_accuracy: 0.0059\n",
      "Epoch 23/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0087 - accuracy: 0.0062 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 24/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0087 - accuracy: 0.0068\n",
      "Epoch 00024: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-10.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0087 - accuracy: 0.0067 - val_loss: 0.0087 - val_accuracy: 0.0057\n",
      "Epoch 25/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0087 - accuracy: 0.0057Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0087 - accuracy: 0.0056 - val_loss: 0.0087 - val_accuracy: 0.0059\n",
      "Epoch 00025: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0686 - accuracy: 0.0455 - val_loss: 0.0231 - val_accuracy: 0.0763\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0238 - accuracy: 0.0900 - val_loss: 0.0220 - val_accuracy: 0.0877\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0226 - accuracy: 0.1046 - val_loss: 0.0214 - val_accuracy: 0.0998\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0219 - accuracy: 0.1213 - val_loss: 0.0212 - val_accuracy: 0.1071\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0214 - accuracy: 0.1266 - val_loss: 0.0210 - val_accuracy: 0.1132\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0209 - accuracy: 0.1323 - val_loss: 0.0208 - val_accuracy: 0.1210\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0207 - accuracy: 0.1387 - val_loss: 0.0207 - val_accuracy: 0.1237\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0203 - accuracy: 0.1476 - val_loss: 0.0205 - val_accuracy: 0.1604\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0200 - accuracy: 0.1528 - val_loss: 0.0204 - val_accuracy: 0.1296\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0198 - accuracy: 0.1576 - val_loss: 0.0203 - val_accuracy: 0.1339\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0194 - accuracy: 0.1661 - val_loss: 0.0203 - val_accuracy: 0.1364\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0192 - accuracy: 0.1793 - val_loss: 0.0205 - val_accuracy: 0.1273\n",
      "Epoch 13/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0190 - accuracy: 0.1792\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0190 - accuracy: 0.1791 - val_loss: 0.0204 - val_accuracy: 0.1328\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0181 - accuracy: 0.1987 - val_loss: 0.0201 - val_accuracy: 0.1346\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0180 - accuracy: 0.1983 - val_loss: 0.0201 - val_accuracy: 0.1405\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0178 - accuracy: 0.2042 - val_loss: 0.0201 - val_accuracy: 0.1376\n",
      "Epoch 17/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0177 - accuracy: 0.2034\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0177 - accuracy: 0.2034 - val_loss: 0.0201 - val_accuracy: 0.1367\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0175 - accuracy: 0.2102 - val_loss: 0.0201 - val_accuracy: 0.1364\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0175 - accuracy: 0.2105 - val_loss: 0.0201 - val_accuracy: 0.1364\n",
      "Epoch 20/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0175 - accuracy: 0.2111\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0175 - accuracy: 0.2112 - val_loss: 0.0202 - val_accuracy: 0.1369\n",
      "Epoch 21/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0175 - accuracy: 0.2090 - val_loss: 0.0201 - val_accuracy: 0.1369\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0175 - accuracy: 0.2102 - val_loss: 0.0201 - val_accuracy: 0.1371\n",
      "Epoch 23/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0175 - accuracy: 0.2102\n",
      "Epoch 00023: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0175 - accuracy: 0.2102 - val_loss: 0.0201 - val_accuracy: 0.1371\n",
      "Epoch 24/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0175 - accuracy: 0.2078 - val_loss: 0.0201 - val_accuracy: 0.1371\n",
      "Epoch 25/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0175 - accuracy: 0.2108Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0175 - accuracy: 0.2109 - val_loss: 0.0201 - val_accuracy: 0.1367\n",
      "Epoch 00025: early stopping\n",
      "SEED:3255 Fold:3\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.2865 - accuracy: 9.6816e-04 - val_loss: 0.0175 - val_accuracy: 2.2784e-04\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0119 - accuracy: 0.0030 - val_loss: 0.0095 - val_accuracy: 0.0043\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0089 - accuracy: 0.0052 - val_loss: 0.0084 - val_accuracy: 0.0057\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0084 - accuracy: 0.0050 - val_loss: 0.0082 - val_accuracy: 0.0055\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0083 - accuracy: 0.0058 - val_loss: 0.0082 - val_accuracy: 0.0064\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0082 - accuracy: 0.0067 - val_loss: 0.0082 - val_accuracy: 0.0062\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - ETA: 0s - loss: 0.0082 - accuracy: 0.0075\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0082 - accuracy: 0.0075 - val_loss: 0.0082 - val_accuracy: 0.0055\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0080 - accuracy: 0.0105 - val_loss: 0.0080 - val_accuracy: 0.0080\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0080 - accuracy: 0.0119 - val_loss: 0.0080 - val_accuracy: 0.0082\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0080 - accuracy: 0.0121 - val_loss: 0.0080 - val_accuracy: 0.0089\n",
      "Epoch 11/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0080 - accuracy: 0.0127\n",
      "Epoch 00011: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0080 - accuracy: 0.0127 - val_loss: 0.0080 - val_accuracy: 0.0091\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0079 - accuracy: 0.0133 - val_loss: 0.0080 - val_accuracy: 0.0093\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0079 - accuracy: 0.0137 - val_loss: 0.0080 - val_accuracy: 0.0093\n",
      "Epoch 14/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0079 - accuracy: 0.0136\n",
      "Epoch 00014: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 2s 13ms/step - loss: 0.0079 - accuracy: 0.0136 - val_loss: 0.0080 - val_accuracy: 0.0100\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0079 - accuracy: 0.0136 - val_loss: 0.0080 - val_accuracy: 0.0098\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0079 - accuracy: 0.0135 - val_loss: 0.0080 - val_accuracy: 0.0098\n",
      "Epoch 17/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0079 - accuracy: 0.0136\n",
      "Epoch 00017: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0079 - accuracy: 0.0136 - val_loss: 0.0080 - val_accuracy: 0.0100\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0079 - accuracy: 0.0134 - val_loss: 0.0080 - val_accuracy: 0.0100\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0079 - accuracy: 0.0137 - val_loss: 0.0080 - val_accuracy: 0.0100\n",
      "Epoch 20/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0079 - accuracy: 0.0141\n",
      "Epoch 00020: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 11ms/step - loss: 0.0079 - accuracy: 0.0141 - val_loss: 0.0080 - val_accuracy: 0.0100\n",
      "Epoch 00020: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.1245 - accuracy: 0.0379 - val_loss: 0.0267 - val_accuracy: 0.0242\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0237 - accuracy: 0.0838 - val_loss: 0.0225 - val_accuracy: 0.1253\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0218 - accuracy: 0.1097 - val_loss: 0.0216 - val_accuracy: 0.0998\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0210 - accuracy: 0.1232 - val_loss: 0.0211 - val_accuracy: 0.1465\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 2s 12ms/step - loss: 0.0205 - accuracy: 0.1360 - val_loss: 0.0207 - val_accuracy: 0.1479\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0201 - accuracy: 0.1435 - val_loss: 0.0207 - val_accuracy: 0.1173\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0197 - accuracy: 0.1506 - val_loss: 0.0205 - val_accuracy: 0.1586\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0194 - accuracy: 0.1656 - val_loss: 0.0204 - val_accuracy: 0.1638\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0190 - accuracy: 0.1682 - val_loss: 0.0203 - val_accuracy: 0.1381\n",
      "Epoch 10/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0186 - accuracy: 0.1873 - val_loss: 0.0204 - val_accuracy: 0.1417\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0182 - accuracy: 0.1997 - val_loss: 0.0205 - val_accuracy: 0.1643\n",
      "Epoch 12/150\n",
      "137/138 [============================>.] - ETA: 0s - loss: 0.0178 - accuracy: 0.2100\n",
      "Epoch 00012: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0178 - accuracy: 0.2100 - val_loss: 0.0206 - val_accuracy: 0.1704\n",
      "Epoch 13/150\n",
      "138/138 [==============================] - 2s 11ms/step - loss: 0.0168 - accuracy: 0.2402 - val_loss: 0.0203 - val_accuracy: 0.1738\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0164 - accuracy: 0.2568 - val_loss: 0.0203 - val_accuracy: 0.1709\n",
      "Epoch 15/150\n",
      "135/138 [============================>.] - ETA: 0s - loss: 0.0162 - accuracy: 0.2561\n",
      "Epoch 00015: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0162 - accuracy: 0.2562 - val_loss: 0.0204 - val_accuracy: 0.1486\n",
      "Epoch 16/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0160 - accuracy: 0.2530 - val_loss: 0.0204 - val_accuracy: 0.1611\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0160 - accuracy: 0.2537 - val_loss: 0.0204 - val_accuracy: 0.1627\n",
      "Epoch 18/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0160 - accuracy: 0.2556\n",
      "Epoch 00018: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0160 - accuracy: 0.2556 - val_loss: 0.0204 - val_accuracy: 0.1620\n",
      "Epoch 19/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0160 - accuracy: 0.2527 - val_loss: 0.0204 - val_accuracy: 0.1654\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0159 - accuracy: 0.2566 - val_loss: 0.0204 - val_accuracy: 0.1645\n",
      "Epoch 21/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0160 - accuracy: 0.2538\n",
      "Epoch 00021: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0160 - accuracy: 0.2541 - val_loss: 0.0204 - val_accuracy: 0.1659\n",
      "Epoch 22/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0160 - accuracy: 0.2538 - val_loss: 0.0204 - val_accuracy: 0.1652\n",
      "Epoch 23/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0160 - accuracy: 0.2567Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0160 - accuracy: 0.2563 - val_loss: 0.0204 - val_accuracy: 0.1654\n",
      "Epoch 00023: early stopping\n",
      "SEED:3255 Fold:4\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.2994 - accuracy: 0.0072 - val_loss: 0.0164 - val_accuracy: 0.0093\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0127 - accuracy: 0.0071 - val_loss: 0.0090 - val_accuracy: 0.0041\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0096 - accuracy: 0.0051 - val_loss: 0.0093 - val_accuracy: 0.0064\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0090 - accuracy: 0.0054 - val_loss: 0.0085 - val_accuracy: 0.0059\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0088 - accuracy: 0.0064 - val_loss: 0.0085 - val_accuracy: 0.0075\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0088 - accuracy: 0.0067 - val_loss: 0.0085 - val_accuracy: 0.0071\n",
      "Epoch 7/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0071\n",
      "Epoch 00007: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0087 - accuracy: 0.0072 - val_loss: 0.0086 - val_accuracy: 0.0075\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0086 - accuracy: 0.0072 - val_loss: 0.0085 - val_accuracy: 0.0087\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 9ms/step - loss: 0.0086 - accuracy: 0.0085 - val_loss: 0.0085 - val_accuracy: 0.0089\n",
      "Epoch 10/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0085 - accuracy: 0.0091\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0091 - val_loss: 0.0085 - val_accuracy: 0.0087\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0091 - val_loss: 0.0085 - val_accuracy: 0.0089\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0085 - val_loss: 0.0085 - val_accuracy: 0.0089\n",
      "Epoch 13/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0085 - accuracy: 0.0083\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0085 - accuracy: 0.0083 - val_loss: 0.0085 - val_accuracy: 0.0091\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0084 - val_loss: 0.0085 - val_accuracy: 0.0091\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0084 - val_loss: 0.0085 - val_accuracy: 0.0091\n",
      "Epoch 16/150\n",
      "132/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0083\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000001111620805e-07.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0086 - accuracy: 0.0083 - val_loss: 0.0085 - val_accuracy: 0.0091\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 10ms/step - loss: 0.0085 - accuracy: 0.0086 - val_loss: 0.0085 - val_accuracy: 0.0089\n",
      "Epoch 18/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0089 - val_loss: 0.0085 - val_accuracy: 0.0091\n",
      "Epoch 19/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0086 - accuracy: 0.0091\n",
      "Epoch 00019: ReduceLROnPlateau reducing learning rate to 1.000000082740371e-08.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0085 - accuracy: 0.0091 - val_loss: 0.0085 - val_accuracy: 0.0089\n",
      "Epoch 20/150\n",
      "138/138 [==============================] - 1s 8ms/step - loss: 0.0085 - accuracy: 0.0089 - val_loss: 0.0085 - val_accuracy: 0.0089\n",
      "Epoch 21/150\n",
      "130/138 [===========================>..] - ETA: 0s - loss: 0.0086 - accuracy: 0.0084Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0086 - accuracy: 0.0084 - val_loss: 0.0085 - val_accuracy: 0.0091\n",
      "Epoch 00021: early stopping\n",
      "Epoch 1/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0910 - accuracy: 0.0533 - val_loss: 0.0238 - val_accuracy: 0.1041\n",
      "Epoch 2/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0230 - accuracy: 0.1154 - val_loss: 0.0218 - val_accuracy: 0.1077\n",
      "Epoch 3/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0214 - accuracy: 0.1293 - val_loss: 0.0211 - val_accuracy: 0.1492\n",
      "Epoch 4/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0205 - accuracy: 0.1535 - val_loss: 0.0206 - val_accuracy: 0.1319\n",
      "Epoch 5/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0198 - accuracy: 0.1682 - val_loss: 0.0207 - val_accuracy: 0.1378\n",
      "Epoch 6/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0191 - accuracy: 0.1823 - val_loss: 0.0205 - val_accuracy: 0.1369\n",
      "Epoch 7/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0182 - accuracy: 0.2066 - val_loss: 0.0204 - val_accuracy: 0.1444\n",
      "Epoch 8/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0173 - accuracy: 0.2307 - val_loss: 0.0203 - val_accuracy: 0.1490\n",
      "Epoch 9/150\n",
      "138/138 [==============================] - 1s 7ms/step - loss: 0.0165 - accuracy: 0.2596 - val_loss: 0.0205 - val_accuracy: 0.1595\n",
      "Epoch 10/150\n",
      "131/138 [===========================>..] - ETA: 0s - loss: 0.0156 - accuracy: 0.2910\n",
      "Epoch 00010: ReduceLROnPlateau reducing learning rate to 0.00010000000474974513.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0156 - accuracy: 0.2896 - val_loss: 0.0207 - val_accuracy: 0.1647\n",
      "Epoch 11/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0141 - accuracy: 0.3450 - val_loss: 0.0206 - val_accuracy: 0.1515\n",
      "Epoch 12/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0137 - accuracy: 0.3628 - val_loss: 0.0207 - val_accuracy: 0.1544\n",
      "Epoch 13/150\n",
      "133/138 [===========================>..] - ETA: 0s - loss: 0.0134 - accuracy: 0.3749\n",
      "Epoch 00013: ReduceLROnPlateau reducing learning rate to 1.0000000474974514e-05.\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0134 - accuracy: 0.3752 - val_loss: 0.0208 - val_accuracy: 0.1513\n",
      "Epoch 14/150\n",
      "138/138 [==============================] - 1s 6ms/step - loss: 0.0131 - accuracy: 0.3816 - val_loss: 0.0208 - val_accuracy: 0.1522\n",
      "Epoch 15/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0131 - accuracy: 0.3829 - val_loss: 0.0208 - val_accuracy: 0.1508\n",
      "Epoch 16/150\n",
      "136/138 [============================>.] - ETA: 0s - loss: 0.0131 - accuracy: 0.3810\n",
      "Epoch 00016: ReduceLROnPlateau reducing learning rate to 1.0000000656873453e-06.\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0131 - accuracy: 0.3814 - val_loss: 0.0208 - val_accuracy: 0.1513\n",
      "Epoch 17/150\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0130 - accuracy: 0.3876 - val_loss: 0.0208 - val_accuracy: 0.1522\n",
      "Epoch 18/150\n",
      "134/138 [============================>.] - ETA: 0s - loss: 0.0131 - accuracy: 0.3874Restoring model weights from the end of the best epoch.\n",
      "138/138 [==============================] - 1s 5ms/step - loss: 0.0131 - accuracy: 0.3870 - val_loss: 0.0208 - val_accuracy: 0.1517\n",
      "Epoch 00018: early stopping\n"
     ]
    }
   ],
   "source": [
    "for seed in [30,58,3255]:\n",
    "    for n,(tr,te) in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True).split(X_train,y_train)):\n",
    "        print(\"SEED:\"+str(seed)+\" Fold:\"+str(n))\n",
    "        \n",
    "        units=hps[seed][n]['units']\n",
    "        activations=hps[seed][n]['activations']\n",
    "        dropouts=hps[seed][n]['dropouts']\n",
    "        \n",
    "        \n",
    "        model2=create_model(X_train.shape[1], units, activations, dropouts, 1e-3,y_train_non.shape[1])\n",
    "        \n",
    "        model2.fit(X_train.values[tr],\n",
    "         y_train_non.values[tr],\n",
    "         batch_size=128,\n",
    "         epochs=150,\n",
    "         validation_data=(X_train.values[te],y_train_non.values[te]),\n",
    "         verbose=1,\n",
    "          callbacks=[\n",
    "              tf.keras.callbacks.ReduceLROnPlateau(\n",
    "                  monitor='val_loss', \n",
    "                  factor=0.1, \n",
    "                  patience=3,\n",
    "                  epsilon = 1e-4, \n",
    "                  mode = 'min',\n",
    "                  verbose=1\n",
    "              )\n",
    "              ,\n",
    "              \n",
    "              tf.keras.callbacks.EarlyStopping(\n",
    "                  monitor='val_loss',\n",
    "                  min_delta=0,\n",
    "                  patience=10,\n",
    "                  mode='auto',\n",
    "                  verbose=1,\n",
    "                  baseline=None,\n",
    "                  restore_best_weights=True\n",
    "              )\n",
    "          ]\n",
    "         )\n",
    "\n",
    "        model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, y_train.shape[1])\n",
    "        for i in range(len(model.layers)-1):\n",
    "            model.layers[i].set_weights(model2.layers[i].get_weights())\n",
    "\n",
    "        model.fit(X_train.values[tr],\n",
    "                 y_train.values[tr],\n",
    "                 batch_size=128,\n",
    "                 epochs=150,\n",
    "                 validation_data=(X_train.values[te],y_train.values[te]),\n",
    "                 verbose=1,\n",
    "                  callbacks=[\n",
    "                      tf.keras.callbacks.ReduceLROnPlateau(\n",
    "                          monitor='val_loss', \n",
    "                          factor=0.1, \n",
    "                          patience=3,\n",
    "                          epsilon = 1e-4, \n",
    "                          mode = 'min',\n",
    "                          verbose=1\n",
    "                      )\n",
    "                      ,\n",
    "\n",
    "                      tf.keras.callbacks.EarlyStopping(\n",
    "                          monitor='val_loss',\n",
    "                          min_delta=0,\n",
    "                          patience=10,\n",
    "                          mode='auto',\n",
    "                          verbose=1,\n",
    "                          baseline=None,\n",
    "                          restore_best_weights=True\n",
    "                      )\n",
    "                  ]\n",
    "                 )\n",
    "        model.save(f'tSEED{seed}FOLD{n}.h5')\n",
    "        model=create_model(X_train.shape[1], units, activations, dropouts, 1e-3, y_train.shape[1])\n",
    "        model.load_weights(f'tSEED{seed}FOLD{n}.h5')\n",
    "        \n",
    "        final_pred+=model.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:31:10.232038Z",
     "iopub.status.busy": "2020-11-26T03:31:10.229898Z",
     "iopub.status.idle": "2020-11-26T03:31:10.232819Z",
     "shell.execute_reply": "2020-11-26T03:31:10.233363Z"
    },
    "papermill": {
     "duration": 8.721648,
     "end_time": "2020-11-26T03:31:10.233510",
     "exception": false,
     "start_time": "2020-11-26T03:31:01.511862",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_pred_transfer=final_pred/15\n",
    "# y_pred=y_pred/2.35"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:31:27.446156Z",
     "iopub.status.busy": "2020-11-26T03:31:27.445117Z",
     "iopub.status.idle": "2020-11-26T03:31:27.447954Z",
     "shell.execute_reply": "2020-11-26T03:31:27.448477Z"
    },
    "papermill": {
     "duration": 8.484797,
     "end_time": "2020-11-26T03:31:27.448648",
     "exception": false,
     "start_time": "2020-11-26T03:31:18.963851",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import log_loss\n",
    "from pytorch_tabnet.metrics import Metric\n",
    "from sklearn.metrics import roc_auc_score, log_loss\n",
    "\n",
    "class LogitsLogLoss(Metric):\n",
    "    \"\"\"\n",
    "    LogLoss with sigmoid applied\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        self._name = \"logits_ll\"\n",
    "        self._maximize = False\n",
    "\n",
    "    def __call__(self, y_true, y_pred):\n",
    "        \"\"\"\n",
    "        Compute LogLoss of predictions.\n",
    "\n",
    "        Parameters\n",
    "        ----------\n",
    "        y_true: np.ndarray\n",
    "            Target matrix or vector\n",
    "        y_score: np.ndarray\n",
    "            Score matrix or vector\n",
    "\n",
    "        Returns\n",
    "        -------\n",
    "            float\n",
    "            LogLoss of predictions vs targets.\n",
    "        \"\"\"\n",
    "        logits = 1 / (1 + np.exp(-y_pred))\n",
    "        aux = (1-y_true)*np.log(1-logits+1e-15) + y_true*np.log(logits+1e-15)\n",
    "        return np.mean(-aux)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:31:46.456362Z",
     "iopub.status.busy": "2020-11-26T03:31:46.455486Z",
     "iopub.status.idle": "2020-11-26T03:31:46.459797Z",
     "shell.execute_reply": "2020-11-26T03:31:46.459192Z"
    },
    "papermill": {
     "duration": 8.864283,
     "end_time": "2020-11-26T03:31:46.459917",
     "exception": false,
     "start_time": "2020-11-26T03:31:37.595634",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "tabnet_params = dict(\n",
    "    n_d = 32,\n",
    "    n_a = 32,\n",
    "    n_steps = 1,\n",
    "    gamma = 1.3,\n",
    "    lambda_sparse = 0,\n",
    "    optimizer_fn = optim.Adam,\n",
    "    optimizer_params = dict(lr = 2e-2, weight_decay = 1e-5),\n",
    "    mask_type = \"entmax\",\n",
    "    scheduler_params = dict(\n",
    "        mode = \"min\", patience = 5, min_lr = 1e-5, factor = 0.9),\n",
    "    scheduler_fn = ReduceLROnPlateau,\n",
    "    seed = 456,\n",
    "    verbose = 10\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T03:32:04.396632Z",
     "iopub.status.busy": "2020-11-26T03:32:04.395362Z",
     "iopub.status.idle": "2020-11-26T05:07:58.860592Z",
     "shell.execute_reply": "2020-11-26T05:07:58.861189Z"
    },
    "papermill": {
     "duration": 5763.708415,
     "end_time": "2020-11-26T05:07:58.861382",
     "exception": false,
     "start_time": "2020-11-26T03:31:55.152967",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=30 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:30 Fold:0\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21095 | val_logits_ll: 0.02737 |  0:00:02s\n",
      "epoch 10 | loss: 0.01829 | val_logits_ll: 0.01803 |  0:00:26s\n",
      "epoch 20 | loss: 0.01748 | val_logits_ll: 0.01775 |  0:00:49s\n",
      "epoch 30 | loss: 0.01736 | val_logits_ll: 0.0174  |  0:01:13s\n",
      "epoch 40 | loss: 0.01712 | val_logits_ll: 0.01748 |  0:01:37s\n",
      "epoch 50 | loss: 0.0171  | val_logits_ll: 0.01716 |  0:02:00s\n",
      "epoch 60 | loss: 0.01687 | val_logits_ll: 0.01698 |  0:02:29s\n",
      "epoch 70 | loss: 0.01668 | val_logits_ll: 0.01689 |  0:02:53s\n",
      "epoch 80 | loss: 0.01649 | val_logits_ll: 0.01676 |  0:03:16s\n",
      "epoch 90 | loss: 0.01639 | val_logits_ll: 0.0167  |  0:03:40s\n",
      "epoch 100| loss: 0.01611 | val_logits_ll: 0.01674 |  0:04:03s\n",
      "epoch 110| loss: 0.01597 | val_logits_ll: 0.01669 |  0:04:27s\n",
      "\n",
      "Early stopping occured at epoch 113 with best_epoch = 93 and best_val_logits_ll = 0.01656\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED30FOLD0.zip\n",
      "SEED:30 Fold:1\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21309 | val_logits_ll: 0.02908 |  0:00:02s\n",
      "epoch 10 | loss: 0.01838 | val_logits_ll: 0.01866 |  0:00:25s\n",
      "epoch 20 | loss: 0.01777 | val_logits_ll: 0.0177  |  0:00:50s\n",
      "epoch 30 | loss: 0.01745 | val_logits_ll: 0.01766 |  0:01:13s\n",
      "epoch 40 | loss: 0.01718 | val_logits_ll: 0.01749 |  0:01:35s\n",
      "epoch 50 | loss: 0.01703 | val_logits_ll: 0.01717 |  0:02:00s\n",
      "epoch 60 | loss: 0.01684 | val_logits_ll: 0.01719 |  0:02:24s\n",
      "epoch 70 | loss: 0.01657 | val_logits_ll: 0.01744 |  0:02:53s\n",
      "epoch 80 | loss: 0.01658 | val_logits_ll: 0.01719 |  0:03:16s\n",
      "epoch 90 | loss: 0.01646 | val_logits_ll: 0.01712 |  0:03:40s\n",
      "epoch 100| loss: 0.01648 | val_logits_ll: 0.01712 |  0:04:04s\n",
      "epoch 110| loss: 0.01624 | val_logits_ll: 0.01692 |  0:04:27s\n",
      "epoch 120| loss: 0.01618 | val_logits_ll: 0.01694 |  0:04:52s\n",
      "epoch 130| loss: 0.01598 | val_logits_ll: 0.01684 |  0:05:20s\n",
      "epoch 140| loss: 0.01588 | val_logits_ll: 0.01682 |  0:05:44s\n",
      "epoch 150| loss: 0.01564 | val_logits_ll: 0.01682 |  0:06:08s\n",
      "epoch 160| loss: 0.01539 | val_logits_ll: 0.01675 |  0:06:31s\n",
      "epoch 170| loss: 0.0151  | val_logits_ll: 0.01688 |  0:06:55s\n",
      "epoch 180| loss: 0.01479 | val_logits_ll: 0.01698 |  0:07:18s\n",
      "\n",
      "Early stopping occured at epoch 180 with best_epoch = 160 and best_val_logits_ll = 0.01675\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED30FOLD1.zip\n",
      "SEED:30 Fold:2\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21312 | val_logits_ll: 0.02926 |  0:00:03s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: RuntimeWarning: overflow encountered in exp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10 | loss: 0.01817 | val_logits_ll: 0.02084 |  0:00:30s\n",
      "epoch 20 | loss: 0.01765 | val_logits_ll: 0.01784 |  0:00:54s\n",
      "epoch 30 | loss: 0.01748 | val_logits_ll: 0.01774 |  0:01:17s\n",
      "epoch 40 | loss: 0.01717 | val_logits_ll: 0.01722 |  0:01:41s\n",
      "epoch 50 | loss: 0.01691 | val_logits_ll: 0.0172  |  0:02:05s\n",
      "epoch 60 | loss: 0.01674 | val_logits_ll: 0.01735 |  0:02:29s\n",
      "epoch 70 | loss: 0.01669 | val_logits_ll: 0.01709 |  0:02:58s\n",
      "epoch 80 | loss: 0.0166  | val_logits_ll: 0.01698 |  0:03:21s\n",
      "epoch 90 | loss: 0.01653 | val_logits_ll: 0.01703 |  0:03:46s\n",
      "epoch 100| loss: 0.01644 | val_logits_ll: 0.01685 |  0:04:09s\n",
      "epoch 110| loss: 0.01618 | val_logits_ll: 0.01689 |  0:04:34s\n",
      "epoch 120| loss: 0.01598 | val_logits_ll: 0.01683 |  0:04:56s\n",
      "epoch 130| loss: 0.0159  | val_logits_ll: 0.01676 |  0:05:26s\n",
      "epoch 140| loss: 0.01564 | val_logits_ll: 0.01681 |  0:05:50s\n",
      "epoch 150| loss: 0.01558 | val_logits_ll: 0.01691 |  0:06:13s\n",
      "epoch 160| loss: 0.01531 | val_logits_ll: 0.01681 |  0:06:37s\n",
      "\n",
      "Early stopping occured at epoch 164 with best_epoch = 144 and best_val_logits_ll = 0.0167\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED30FOLD2.zip\n",
      "SEED:30 Fold:3\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21013 | val_logits_ll: 0.02962 |  0:00:02s\n",
      "epoch 10 | loss: 0.01815 | val_logits_ll: 0.02185 |  0:00:24s\n",
      "epoch 20 | loss: 0.0177  | val_logits_ll: 0.0196  |  0:00:55s\n",
      "epoch 30 | loss: 0.01752 | val_logits_ll: 0.0176  |  0:01:18s\n",
      "epoch 40 | loss: 0.01736 | val_logits_ll: 0.01742 |  0:01:42s\n",
      "epoch 50 | loss: 0.01718 | val_logits_ll: 0.01716 |  0:02:05s\n",
      "epoch 60 | loss: 0.01693 | val_logits_ll: 0.01704 |  0:02:28s\n",
      "epoch 70 | loss: 0.01686 | val_logits_ll: 0.01725 |  0:02:52s\n",
      "epoch 80 | loss: 0.01665 | val_logits_ll: 0.01706 |  0:03:20s\n",
      "epoch 90 | loss: 0.01657 | val_logits_ll: 0.01694 |  0:03:44s\n",
      "epoch 100| loss: 0.01648 | val_logits_ll: 0.01712 |  0:04:07s\n",
      "epoch 110| loss: 0.01625 | val_logits_ll: 0.01691 |  0:04:30s\n",
      "epoch 120| loss: 0.01616 | val_logits_ll: 0.0168  |  0:04:55s\n",
      "epoch 130| loss: 0.01586 | val_logits_ll: 0.01683 |  0:05:18s\n",
      "epoch 140| loss: 0.01578 | val_logits_ll: 0.01683 |  0:05:48s\n",
      "epoch 150| loss: 0.01548 | val_logits_ll: 0.01672 |  0:06:11s\n",
      "\n",
      "Early stopping occured at epoch 152 with best_epoch = 132 and best_val_logits_ll = 0.01666\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED30FOLD3.zip\n",
      "SEED:30 Fold:4\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21096 | val_logits_ll: 0.02963 |  0:00:02s\n",
      "epoch 10 | loss: 0.01869 | val_logits_ll: 0.01919 |  0:00:27s\n",
      "epoch 20 | loss: 0.01772 | val_logits_ll: 0.01793 |  0:00:50s\n",
      "epoch 30 | loss: 0.01737 | val_logits_ll: 0.0179  |  0:01:13s\n",
      "epoch 40 | loss: 0.01704 | val_logits_ll: 0.01724 |  0:01:37s\n",
      "epoch 50 | loss: 0.0168  | val_logits_ll: 0.01703 |  0:02:06s\n",
      "epoch 60 | loss: 0.01662 | val_logits_ll: 0.01694 |  0:02:31s\n",
      "epoch 70 | loss: 0.01638 | val_logits_ll: 0.01696 |  0:02:54s\n",
      "epoch 80 | loss: 0.01628 | val_logits_ll: 0.01677 |  0:03:17s\n",
      "epoch 90 | loss: 0.01614 | val_logits_ll: 0.0166  |  0:03:41s\n",
      "epoch 100| loss: 0.01586 | val_logits_ll: 0.01667 |  0:04:04s\n",
      "epoch 110| loss: 0.01558 | val_logits_ll: 0.01665 |  0:04:35s\n",
      "\n",
      "Early stopping occured at epoch 110 with best_epoch = 90 and best_val_logits_ll = 0.0166\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED30FOLD4.zip\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=58 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:58 Fold:0\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21178 | val_logits_ll: 0.02925 |  0:00:02s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: RuntimeWarning: overflow encountered in exp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10 | loss: 0.01828 | val_logits_ll: 0.0182  |  0:00:25s\n",
      "epoch 20 | loss: 0.01764 | val_logits_ll: 0.01856 |  0:00:49s\n",
      "epoch 30 | loss: 0.01744 | val_logits_ll: 0.0175  |  0:01:13s\n",
      "epoch 40 | loss: 0.01709 | val_logits_ll: 0.0173  |  0:01:36s\n",
      "epoch 50 | loss: 0.0169  | val_logits_ll: 0.01713 |  0:02:00s\n",
      "epoch 60 | loss: 0.0167  | val_logits_ll: 0.0171  |  0:02:29s\n",
      "epoch 70 | loss: 0.01647 | val_logits_ll: 0.01704 |  0:02:53s\n",
      "epoch 80 | loss: 0.01635 | val_logits_ll: 0.01686 |  0:03:17s\n",
      "epoch 90 | loss: 0.01632 | val_logits_ll: 0.01713 |  0:03:41s\n",
      "epoch 100| loss: 0.0162  | val_logits_ll: 0.01696 |  0:04:05s\n",
      "epoch 110| loss: 0.01592 | val_logits_ll: 0.01687 |  0:04:29s\n",
      "epoch 120| loss: 0.01577 | val_logits_ll: 0.01674 |  0:04:59s\n",
      "epoch 130| loss: 0.01555 | val_logits_ll: 0.01669 |  0:05:22s\n",
      "epoch 140| loss: 0.01544 | val_logits_ll: 0.01672 |  0:05:46s\n",
      "epoch 150| loss: 0.0151  | val_logits_ll: 0.01674 |  0:06:09s\n",
      "epoch 160| loss: 0.01487 | val_logits_ll: 0.01678 |  0:06:33s\n",
      "\n",
      "Early stopping occured at epoch 162 with best_epoch = 142 and best_val_logits_ll = 0.0166\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED58FOLD0.zip\n",
      "SEED:58 Fold:1\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21053 | val_logits_ll: 0.0315  |  0:00:02s\n",
      "epoch 10 | loss: 0.01827 | val_logits_ll: 0.02037 |  0:00:25s\n",
      "epoch 20 | loss: 0.01761 | val_logits_ll: 0.01809 |  0:00:55s\n",
      "epoch 30 | loss: 0.01729 | val_logits_ll: 0.0179  |  0:01:19s\n",
      "epoch 40 | loss: 0.0171  | val_logits_ll: 0.01739 |  0:01:42s\n",
      "epoch 50 | loss: 0.01692 | val_logits_ll: 0.01728 |  0:02:07s\n",
      "epoch 60 | loss: 0.01669 | val_logits_ll: 0.01716 |  0:02:30s\n",
      "epoch 70 | loss: 0.01649 | val_logits_ll: 0.0171  |  0:02:53s\n",
      "epoch 80 | loss: 0.0162  | val_logits_ll: 0.01695 |  0:03:23s\n",
      "epoch 90 | loss: 0.01606 | val_logits_ll: 0.01705 |  0:03:47s\n",
      "epoch 100| loss: 0.01583 | val_logits_ll: 0.01694 |  0:04:12s\n",
      "epoch 110| loss: 0.01554 | val_logits_ll: 0.0169  |  0:04:35s\n",
      "epoch 120| loss: 0.01527 | val_logits_ll: 0.01706 |  0:04:59s\n",
      "epoch 130| loss: 0.01491 | val_logits_ll: 0.0171  |  0:05:23s\n",
      "epoch 140| loss: 0.0146  | val_logits_ll: 0.01711 |  0:05:52s\n",
      "\n",
      "Early stopping occured at epoch 143 with best_epoch = 123 and best_val_logits_ll = 0.01685\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED58FOLD1.zip\n",
      "SEED:58 Fold:2\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21076 | val_logits_ll: 0.02867 |  0:00:02s\n",
      "epoch 10 | loss: 0.01852 | val_logits_ll: 0.01854 |  0:00:25s\n",
      "epoch 20 | loss: 0.01755 | val_logits_ll: 0.01867 |  0:00:49s\n",
      "epoch 30 | loss: 0.01733 | val_logits_ll: 0.01721 |  0:01:14s\n",
      "epoch 40 | loss: 0.01719 | val_logits_ll: 0.01721 |  0:01:37s\n",
      "epoch 50 | loss: 0.0169  | val_logits_ll: 0.01695 |  0:02:08s\n",
      "epoch 60 | loss: 0.01674 | val_logits_ll: 0.017   |  0:02:31s\n",
      "epoch 70 | loss: 0.01667 | val_logits_ll: 0.01691 |  0:02:55s\n",
      "epoch 80 | loss: 0.01649 | val_logits_ll: 0.01695 |  0:03:18s\n",
      "epoch 90 | loss: 0.01629 | val_logits_ll: 0.0169  |  0:03:41s\n",
      "epoch 100| loss: 0.01615 | val_logits_ll: 0.0167  |  0:04:06s\n",
      "epoch 110| loss: 0.01604 | val_logits_ll: 0.01675 |  0:04:35s\n",
      "epoch 120| loss: 0.0158  | val_logits_ll: 0.01678 |  0:05:00s\n",
      "epoch 130| loss: 0.0157  | val_logits_ll: 0.01665 |  0:05:22s\n",
      "epoch 140| loss: 0.01534 | val_logits_ll: 0.0167  |  0:05:45s\n",
      "epoch 150| loss: 0.01504 | val_logits_ll: 0.01675 |  0:06:10s\n",
      "\n",
      "Early stopping occured at epoch 153 with best_epoch = 133 and best_val_logits_ll = 0.0166\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED58FOLD2.zip\n",
      "SEED:58 Fold:3\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21037 | val_logits_ll: 0.0293  |  0:00:02s\n",
      "epoch 10 | loss: 0.01852 | val_logits_ll: 0.0187  |  0:00:25s\n",
      "epoch 20 | loss: 0.01766 | val_logits_ll: 0.01775 |  0:00:56s\n",
      "epoch 30 | loss: 0.01739 | val_logits_ll: 0.01735 |  0:01:19s\n",
      "epoch 40 | loss: 0.01744 | val_logits_ll: 0.01743 |  0:01:43s\n",
      "epoch 50 | loss: 0.01717 | val_logits_ll: 0.01729 |  0:02:06s\n",
      "epoch 60 | loss: 0.01714 | val_logits_ll: 0.01729 |  0:02:30s\n",
      "epoch 70 | loss: 0.01709 | val_logits_ll: 0.01714 |  0:02:54s\n",
      "epoch 80 | loss: 0.01693 | val_logits_ll: 0.01716 |  0:03:23s\n",
      "epoch 90 | loss: 0.01671 | val_logits_ll: 0.01695 |  0:03:47s\n",
      "epoch 100| loss: 0.01661 | val_logits_ll: 0.01695 |  0:04:10s\n",
      "epoch 110| loss: 0.0165  | val_logits_ll: 0.01695 |  0:04:34s\n",
      "epoch 120| loss: 0.0163  | val_logits_ll: 0.01679 |  0:04:58s\n",
      "epoch 130| loss: 0.01622 | val_logits_ll: 0.01679 |  0:05:21s\n",
      "epoch 140| loss: 0.01587 | val_logits_ll: 0.01687 |  0:05:52s\n",
      "epoch 150| loss: 0.01594 | val_logits_ll: 0.01671 |  0:06:16s\n",
      "epoch 160| loss: 0.01572 | val_logits_ll: 0.01678 |  0:06:40s\n",
      "epoch 170| loss: 0.0155  | val_logits_ll: 0.01677 |  0:07:03s\n",
      "epoch 180| loss: 0.01528 | val_logits_ll: 0.01676 |  0:07:26s\n",
      "\n",
      "Early stopping occured at epoch 184 with best_epoch = 164 and best_val_logits_ll = 0.0166\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED58FOLD3.zip\n",
      "SEED:58 Fold:4\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21081 | val_logits_ll: 0.02922 |  0:00:04s\n",
      "epoch 10 | loss: 0.0183  | val_logits_ll: 0.01986 |  0:00:31s\n",
      "epoch 20 | loss: 0.01747 | val_logits_ll: 0.019   |  0:00:55s\n",
      "epoch 30 | loss: 0.01732 | val_logits_ll: 0.01742 |  0:01:18s\n",
      "epoch 40 | loss: 0.01693 | val_logits_ll: 0.01715 |  0:01:42s\n",
      "epoch 50 | loss: 0.01696 | val_logits_ll: 0.01714 |  0:02:06s\n",
      "epoch 60 | loss: 0.01673 | val_logits_ll: 0.01705 |  0:02:30s\n",
      "epoch 70 | loss: 0.01662 | val_logits_ll: 0.01699 |  0:02:59s\n",
      "epoch 80 | loss: 0.01646 | val_logits_ll: 0.01707 |  0:03:23s\n",
      "epoch 90 | loss: 0.01642 | val_logits_ll: 0.0169  |  0:03:47s\n",
      "epoch 100| loss: 0.01614 | val_logits_ll: 0.01691 |  0:04:10s\n",
      "epoch 110| loss: 0.01615 | val_logits_ll: 0.01683 |  0:04:34s\n",
      "epoch 120| loss: 0.01605 | val_logits_ll: 0.01673 |  0:04:58s\n",
      "epoch 130| loss: 0.01587 | val_logits_ll: 0.01669 |  0:05:27s\n",
      "\n",
      "Early stopping occured at epoch 137 with best_epoch = 117 and best_val_logits_ll = 0.01665\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED58FOLD4.zip\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/sklearn/utils/validation.py:70: FutureWarning: Pass shuffle=True, random_state=3255 as keyword args. From version 0.25 passing these as positional arguments will result in an error\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SEED:3255 Fold:0\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21276 | val_logits_ll: 0.02933 |  0:00:02s\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/conda/lib/python3.7/site-packages/ipykernel_launcher.py:30: RuntimeWarning: overflow encountered in exp\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 10 | loss: 0.01837 | val_logits_ll: 0.0184  |  0:00:26s\n",
      "epoch 20 | loss: 0.01765 | val_logits_ll: 0.01866 |  0:00:50s\n",
      "epoch 30 | loss: 0.01737 | val_logits_ll: 0.01769 |  0:01:15s\n",
      "epoch 40 | loss: 0.01734 | val_logits_ll: 0.01744 |  0:01:38s\n",
      "epoch 50 | loss: 0.01716 | val_logits_ll: 0.01731 |  0:02:07s\n",
      "epoch 60 | loss: 0.017   | val_logits_ll: 0.01733 |  0:02:32s\n",
      "epoch 70 | loss: 0.01676 | val_logits_ll: 0.01718 |  0:02:55s\n",
      "epoch 80 | loss: 0.01669 | val_logits_ll: 0.0171  |  0:03:19s\n",
      "epoch 90 | loss: 0.0165  | val_logits_ll: 0.01705 |  0:03:43s\n",
      "epoch 100| loss: 0.0163  | val_logits_ll: 0.01689 |  0:04:07s\n",
      "epoch 110| loss: 0.01609 | val_logits_ll: 0.01679 |  0:04:37s\n",
      "epoch 120| loss: 0.01603 | val_logits_ll: 0.01676 |  0:05:00s\n",
      "epoch 130| loss: 0.01573 | val_logits_ll: 0.01682 |  0:05:24s\n",
      "epoch 140| loss: 0.0156  | val_logits_ll: 0.01672 |  0:05:47s\n",
      "epoch 150| loss: 0.01541 | val_logits_ll: 0.01678 |  0:06:12s\n",
      "epoch 160| loss: 0.01508 | val_logits_ll: 0.01691 |  0:06:36s\n",
      "\n",
      "Early stopping occured at epoch 160 with best_epoch = 140 and best_val_logits_ll = 0.01672\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED3255FOLD0.zip\n",
      "SEED:3255 Fold:1\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21297 | val_logits_ll: 0.02954 |  0:00:03s\n",
      "epoch 10 | loss: 0.01835 | val_logits_ll: 0.01879 |  0:00:30s\n",
      "epoch 20 | loss: 0.01777 | val_logits_ll: 0.01831 |  0:00:54s\n",
      "epoch 30 | loss: 0.01747 | val_logits_ll: 0.01745 |  0:01:17s\n",
      "epoch 40 | loss: 0.0173  | val_logits_ll: 0.01725 |  0:01:42s\n",
      "epoch 50 | loss: 0.01732 | val_logits_ll: 0.01738 |  0:02:05s\n",
      "epoch 60 | loss: 0.01696 | val_logits_ll: 0.01712 |  0:02:29s\n",
      "epoch 70 | loss: 0.01682 | val_logits_ll: 0.01712 |  0:02:58s\n",
      "epoch 80 | loss: 0.0166  | val_logits_ll: 0.01699 |  0:03:21s\n",
      "epoch 90 | loss: 0.0165  | val_logits_ll: 0.01694 |  0:03:46s\n",
      "epoch 100| loss: 0.01629 | val_logits_ll: 0.01699 |  0:04:09s\n",
      "epoch 110| loss: 0.01624 | val_logits_ll: 0.01676 |  0:04:34s\n",
      "epoch 120| loss: 0.01619 | val_logits_ll: 0.01669 |  0:04:57s\n",
      "epoch 130| loss: 0.01604 | val_logits_ll: 0.01681 |  0:05:26s\n",
      "epoch 140| loss: 0.01579 | val_logits_ll: 0.01686 |  0:05:51s\n",
      "epoch 150| loss: 0.01574 | val_logits_ll: 0.01668 |  0:06:14s\n",
      "epoch 160| loss: 0.01559 | val_logits_ll: 0.01673 |  0:06:39s\n",
      "epoch 170| loss: 0.01538 | val_logits_ll: 0.01666 |  0:07:02s\n",
      "epoch 180| loss: 0.01513 | val_logits_ll: 0.01669 |  0:07:26s\n",
      "epoch 190| loss: 0.01482 | val_logits_ll: 0.01681 |  0:07:55s\n",
      "\n",
      "Early stopping occured at epoch 191 with best_epoch = 171 and best_val_logits_ll = 0.01658\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED3255FOLD1.zip\n",
      "SEED:3255 Fold:2\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21076 | val_logits_ll: 0.02835 |  0:00:02s\n",
      "epoch 10 | loss: 0.0184  | val_logits_ll: 0.0182  |  0:00:24s\n",
      "epoch 20 | loss: 0.0174  | val_logits_ll: 0.01793 |  0:00:49s\n",
      "epoch 30 | loss: 0.01705 | val_logits_ll: 0.01722 |  0:01:12s\n",
      "epoch 40 | loss: 0.01704 | val_logits_ll: 0.017   |  0:01:37s\n",
      "epoch 50 | loss: 0.01678 | val_logits_ll: 0.01708 |  0:02:00s\n",
      "epoch 60 | loss: 0.01653 | val_logits_ll: 0.01692 |  0:02:30s\n",
      "epoch 70 | loss: 0.01646 | val_logits_ll: 0.01686 |  0:02:54s\n",
      "epoch 80 | loss: 0.01634 | val_logits_ll: 0.01683 |  0:03:17s\n",
      "epoch 90 | loss: 0.01626 | val_logits_ll: 0.01678 |  0:03:42s\n",
      "epoch 100| loss: 0.01609 | val_logits_ll: 0.01667 |  0:04:05s\n",
      "epoch 110| loss: 0.01584 | val_logits_ll: 0.01678 |  0:04:29s\n",
      "epoch 120| loss: 0.01564 | val_logits_ll: 0.01675 |  0:04:59s\n",
      "epoch 130| loss: 0.01553 | val_logits_ll: 0.01674 |  0:05:22s\n",
      "epoch 140| loss: 0.01528 | val_logits_ll: 0.01669 |  0:05:47s\n",
      "epoch 150| loss: 0.01507 | val_logits_ll: 0.01696 |  0:06:10s\n",
      "\n",
      "Early stopping occured at epoch 158 with best_epoch = 138 and best_val_logits_ll = 0.01663\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED3255FOLD2.zip\n",
      "SEED:3255 Fold:3\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.20883 | val_logits_ll: 0.03005 |  0:00:02s\n",
      "epoch 10 | loss: 0.01856 | val_logits_ll: 0.0187  |  0:00:26s\n",
      "epoch 20 | loss: 0.0177  | val_logits_ll: 0.01811 |  0:00:54s\n",
      "epoch 30 | loss: 0.01728 | val_logits_ll: 0.01756 |  0:01:18s\n",
      "epoch 40 | loss: 0.01722 | val_logits_ll: 0.01727 |  0:01:42s\n",
      "epoch 50 | loss: 0.01696 | val_logits_ll: 0.01721 |  0:02:06s\n",
      "epoch 60 | loss: 0.01689 | val_logits_ll: 0.0171  |  0:02:29s\n",
      "epoch 70 | loss: 0.01683 | val_logits_ll: 0.01711 |  0:02:52s\n",
      "epoch 80 | loss: 0.01668 | val_logits_ll: 0.01703 |  0:03:23s\n",
      "epoch 90 | loss: 0.01646 | val_logits_ll: 0.01691 |  0:03:46s\n",
      "epoch 100| loss: 0.01633 | val_logits_ll: 0.0168  |  0:04:11s\n",
      "epoch 110| loss: 0.01618 | val_logits_ll: 0.01676 |  0:04:33s\n",
      "epoch 120| loss: 0.01594 | val_logits_ll: 0.01689 |  0:04:57s\n",
      "epoch 130| loss: 0.01587 | val_logits_ll: 0.01674 |  0:05:22s\n",
      "epoch 140| loss: 0.01559 | val_logits_ll: 0.01672 |  0:05:50s\n",
      "epoch 150| loss: 0.01537 | val_logits_ll: 0.01673 |  0:06:16s\n",
      "epoch 160| loss: 0.01504 | val_logits_ll: 0.01668 |  0:06:38s\n",
      "\n",
      "Early stopping occured at epoch 167 with best_epoch = 147 and best_val_logits_ll = 0.01659\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED3255FOLD3.zip\n",
      "SEED:3255 Fold:4\n",
      "Device used : cuda\n",
      "epoch 0  | loss: 0.21092 | val_logits_ll: 0.02848 |  0:00:02s\n",
      "epoch 10 | loss: 0.01862 | val_logits_ll: 0.01928 |  0:00:26s\n",
      "epoch 20 | loss: 0.01776 | val_logits_ll: 0.01771 |  0:00:50s\n",
      "epoch 30 | loss: 0.01734 | val_logits_ll: 0.01744 |  0:01:20s\n",
      "epoch 40 | loss: 0.01715 | val_logits_ll: 0.01759 |  0:01:44s\n",
      "epoch 50 | loss: 0.01706 | val_logits_ll: 0.0173  |  0:02:09s\n",
      "epoch 60 | loss: 0.01695 | val_logits_ll: 0.0172  |  0:02:32s\n",
      "epoch 70 | loss: 0.01665 | val_logits_ll: 0.01705 |  0:02:55s\n",
      "epoch 80 | loss: 0.01645 | val_logits_ll: 0.01712 |  0:03:20s\n",
      "epoch 90 | loss: 0.01624 | val_logits_ll: 0.01689 |  0:03:48s\n",
      "epoch 100| loss: 0.01612 | val_logits_ll: 0.01683 |  0:04:13s\n",
      "epoch 110| loss: 0.016   | val_logits_ll: 0.01672 |  0:04:35s\n",
      "epoch 120| loss: 0.01595 | val_logits_ll: 0.01689 |  0:05:00s\n",
      "epoch 130| loss: 0.01575 | val_logits_ll: 0.01699 |  0:05:24s\n",
      "\n",
      "Early stopping occured at epoch 130 with best_epoch = 110 and best_val_logits_ll = 0.01672\n",
      "Best weights from best epoch are automatically used!\n",
      "Successfully saved model at tabSEED3255FOLD4.zip\n"
     ]
    }
   ],
   "source": [
    "final_pred=submission.iloc[:,1:].copy()\n",
    "final_pred.loc[:,:]=0\n",
    "\n",
    "for seed in [30,58,3255]:\n",
    "    for n,(tr,te) in enumerate(MultilabelStratifiedKFold(n_splits=5, random_state=seed, shuffle=True).split(X_train,y_train)):\n",
    "        print(\"SEED:\"+str(seed)+\" Fold:\"+str(n))\n",
    "        \n",
    "        model = TabNetRegressor(**tabnet_params)\n",
    "        \n",
    "        model.fit(\n",
    "        X_train = X_train.values[tr],\n",
    "        y_train = y_train.values[tr],\n",
    "        eval_set = [(X_train.values[te],y_train.values[te])],\n",
    "        eval_name = [\"val\"],\n",
    "        eval_metric = [\"logits_ll\"],\n",
    "        max_epochs = 200,\n",
    "        patience = 20,\n",
    "        batch_size = 512, \n",
    "        virtual_batch_size = 32,\n",
    "        num_workers = 1,\n",
    "        drop_last = False,\n",
    "        # To use binary cross entropy because this is not a regression problem\n",
    "        loss_fn = F.binary_cross_entropy_with_logits)\n",
    "        \n",
    " \n",
    "        model.save_model(f\"tabSEED{seed}FOLD{n}\")\n",
    "        \n",
    "        final_pred+=1 / (1 + np.exp(-model.predict(X_test.values)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:08:16.980507Z",
     "iopub.status.busy": "2020-11-26T05:08:16.979626Z",
     "iopub.status.idle": "2020-11-26T05:08:16.983768Z",
     "shell.execute_reply": "2020-11-26T05:08:16.983122Z"
    },
    "papermill": {
     "duration": 9.269606,
     "end_time": "2020-11-26T05:08:16.983887",
     "exception": false,
     "start_time": "2020-11-26T05:08:07.714281",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_pred_tabnet=final_pred/15"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:08:35.087807Z",
     "iopub.status.busy": "2020-11-26T05:08:35.086533Z",
     "iopub.status.idle": "2020-11-26T05:08:35.108075Z",
     "shell.execute_reply": "2020-11-26T05:08:35.107170Z"
    },
    "papermill": {
     "duration": 9.502841,
     "end_time": "2020-11-26T05:08:35.108243",
     "exception": false,
     "start_time": "2020-11-26T05:08:25.605402",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "y_pred_final=y_pred * 0.25 + y_pred_transfer * 0.25 + y_pred_tabnet * 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:08:53.741297Z",
     "iopub.status.busy": "2020-11-26T05:08:53.719718Z",
     "iopub.status.idle": "2020-11-26T05:08:53.746700Z",
     "shell.execute_reply": "2020-11-26T05:08:53.747414Z"
    },
    "papermill": {
     "duration": 9.273304,
     "end_time": "2020-11-26T05:08:53.747606",
     "exception": false,
     "start_time": "2020-11-26T05:08:44.474302",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sig_id</th>\n",
       "      <th>5-alpha_reductase_inhibitor</th>\n",
       "      <th>11-beta-hsd1_inhibitor</th>\n",
       "      <th>acat_inhibitor</th>\n",
       "      <th>acetylcholine_receptor_agonist</th>\n",
       "      <th>acetylcholine_receptor_antagonist</th>\n",
       "      <th>acetylcholinesterase_inhibitor</th>\n",
       "      <th>adenosine_receptor_agonist</th>\n",
       "      <th>adenosine_receptor_antagonist</th>\n",
       "      <th>adenylyl_cyclase_activator</th>\n",
       "      <th>...</th>\n",
       "      <th>tropomyosin_receptor_kinase_inhibitor</th>\n",
       "      <th>trpv_agonist</th>\n",
       "      <th>trpv_antagonist</th>\n",
       "      <th>tubulin_inhibitor</th>\n",
       "      <th>tyrosine_kinase_inhibitor</th>\n",
       "      <th>ubiquitin_specific_protease_inhibitor</th>\n",
       "      <th>vegfr_inhibitor</th>\n",
       "      <th>vitamin_b</th>\n",
       "      <th>vitamin_d_receptor_agonist</th>\n",
       "      <th>wnt_inhibitor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_0004d9e33</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_001897cda</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_002429b5b</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_00276f245</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_0027f1083</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "      <td>0.5</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 207 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         sig_id  5-alpha_reductase_inhibitor  11-beta-hsd1_inhibitor  \\\n",
       "0  id_0004d9e33                          0.5                     0.5   \n",
       "1  id_001897cda                          0.5                     0.5   \n",
       "2  id_002429b5b                          0.5                     0.5   \n",
       "3  id_00276f245                          0.5                     0.5   \n",
       "4  id_0027f1083                          0.5                     0.5   \n",
       "\n",
       "   acat_inhibitor  acetylcholine_receptor_agonist  \\\n",
       "0             0.5                             0.5   \n",
       "1             0.5                             0.5   \n",
       "2             0.5                             0.5   \n",
       "3             0.5                             0.5   \n",
       "4             0.5                             0.5   \n",
       "\n",
       "   acetylcholine_receptor_antagonist  acetylcholinesterase_inhibitor  \\\n",
       "0                                0.5                             0.5   \n",
       "1                                0.5                             0.5   \n",
       "2                                0.5                             0.5   \n",
       "3                                0.5                             0.5   \n",
       "4                                0.5                             0.5   \n",
       "\n",
       "   adenosine_receptor_agonist  adenosine_receptor_antagonist  \\\n",
       "0                         0.5                            0.5   \n",
       "1                         0.5                            0.5   \n",
       "2                         0.5                            0.5   \n",
       "3                         0.5                            0.5   \n",
       "4                         0.5                            0.5   \n",
       "\n",
       "   adenylyl_cyclase_activator  ...  tropomyosin_receptor_kinase_inhibitor  \\\n",
       "0                         0.5  ...                                    0.5   \n",
       "1                         0.5  ...                                    0.5   \n",
       "2                         0.5  ...                                    0.5   \n",
       "3                         0.5  ...                                    0.5   \n",
       "4                         0.5  ...                                    0.5   \n",
       "\n",
       "   trpv_agonist  trpv_antagonist  tubulin_inhibitor  \\\n",
       "0           0.5              0.5                0.5   \n",
       "1           0.5              0.5                0.5   \n",
       "2           0.5              0.5                0.5   \n",
       "3           0.5              0.5                0.5   \n",
       "4           0.5              0.5                0.5   \n",
       "\n",
       "   tyrosine_kinase_inhibitor  ubiquitin_specific_protease_inhibitor  \\\n",
       "0                        0.5                                    0.5   \n",
       "1                        0.5                                    0.5   \n",
       "2                        0.5                                    0.5   \n",
       "3                        0.5                                    0.5   \n",
       "4                        0.5                                    0.5   \n",
       "\n",
       "   vegfr_inhibitor  vitamin_b  vitamin_d_receptor_agonist  wnt_inhibitor  \n",
       "0              0.5        0.5                         0.5            0.5  \n",
       "1              0.5        0.5                         0.5            0.5  \n",
       "2              0.5        0.5                         0.5            0.5  \n",
       "3              0.5        0.5                         0.5            0.5  \n",
       "4              0.5        0.5                         0.5            0.5  \n",
       "\n",
       "[5 rows x 207 columns]"
      ]
     },
     "execution_count": 47,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:09:11.015547Z",
     "iopub.status.busy": "2020-11-26T05:09:11.014453Z",
     "iopub.status.idle": "2020-11-26T05:09:11.124438Z",
     "shell.execute_reply": "2020-11-26T05:09:11.122825Z"
    },
    "papermill": {
     "duration": 8.613465,
     "end_time": "2020-11-26T05:09:11.124680",
     "exception": false,
     "start_time": "2020-11-26T05:09:02.511215",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission.iloc[:,1:]=y_pred_final"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:09:29.195408Z",
     "iopub.status.busy": "2020-11-26T05:09:29.190695Z",
     "iopub.status.idle": "2020-11-26T05:09:29.199779Z",
     "shell.execute_reply": "2020-11-26T05:09:29.199165Z"
    },
    "papermill": {
     "duration": 8.904328,
     "end_time": "2020-11-26T05:09:29.199911",
     "exception": false,
     "start_time": "2020-11-26T05:09:20.295583",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission.loc[x_test_checkpoint[\"cp_type\"]=='ctl_vehicle','5-alpha_reductase_inhibitor':]=0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:09:47.769009Z",
     "iopub.status.busy": "2020-11-26T05:09:47.767793Z",
     "iopub.status.idle": "2020-11-26T05:09:47.774675Z",
     "shell.execute_reply": "2020-11-26T05:09:47.775781Z"
    },
    "papermill": {
     "duration": 8.909848,
     "end_time": "2020-11-26T05:09:47.775961",
     "exception": false,
     "start_time": "2020-11-26T05:09:38.866113",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sig_id</th>\n",
       "      <th>5-alpha_reductase_inhibitor</th>\n",
       "      <th>11-beta-hsd1_inhibitor</th>\n",
       "      <th>acat_inhibitor</th>\n",
       "      <th>acetylcholine_receptor_agonist</th>\n",
       "      <th>acetylcholine_receptor_antagonist</th>\n",
       "      <th>acetylcholinesterase_inhibitor</th>\n",
       "      <th>adenosine_receptor_agonist</th>\n",
       "      <th>adenosine_receptor_antagonist</th>\n",
       "      <th>adenylyl_cyclase_activator</th>\n",
       "      <th>...</th>\n",
       "      <th>tropomyosin_receptor_kinase_inhibitor</th>\n",
       "      <th>trpv_agonist</th>\n",
       "      <th>trpv_antagonist</th>\n",
       "      <th>tubulin_inhibitor</th>\n",
       "      <th>tyrosine_kinase_inhibitor</th>\n",
       "      <th>ubiquitin_specific_protease_inhibitor</th>\n",
       "      <th>vegfr_inhibitor</th>\n",
       "      <th>vitamin_b</th>\n",
       "      <th>vitamin_d_receptor_agonist</th>\n",
       "      <th>wnt_inhibitor</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>id_0004d9e33</td>\n",
       "      <td>0.000750</td>\n",
       "      <td>0.001091</td>\n",
       "      <td>0.002089</td>\n",
       "      <td>0.019748</td>\n",
       "      <td>0.018584</td>\n",
       "      <td>0.004491</td>\n",
       "      <td>0.002085</td>\n",
       "      <td>0.005966</td>\n",
       "      <td>0.000325</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001072</td>\n",
       "      <td>0.001195</td>\n",
       "      <td>0.004213</td>\n",
       "      <td>0.001195</td>\n",
       "      <td>0.000634</td>\n",
       "      <td>0.000711</td>\n",
       "      <td>0.000899</td>\n",
       "      <td>0.001795</td>\n",
       "      <td>0.005004</td>\n",
       "      <td>0.001838</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>id_001897cda</td>\n",
       "      <td>0.000415</td>\n",
       "      <td>0.000833</td>\n",
       "      <td>0.001801</td>\n",
       "      <td>0.002423</td>\n",
       "      <td>0.001488</td>\n",
       "      <td>0.001938</td>\n",
       "      <td>0.002919</td>\n",
       "      <td>0.012416</td>\n",
       "      <td>0.006070</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000697</td>\n",
       "      <td>0.001371</td>\n",
       "      <td>0.003716</td>\n",
       "      <td>0.000455</td>\n",
       "      <td>0.010757</td>\n",
       "      <td>0.000635</td>\n",
       "      <td>0.020449</td>\n",
       "      <td>0.001069</td>\n",
       "      <td>0.005172</td>\n",
       "      <td>0.003133</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>id_002429b5b</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>id_00276f245</td>\n",
       "      <td>0.000664</td>\n",
       "      <td>0.000753</td>\n",
       "      <td>0.002374</td>\n",
       "      <td>0.009352</td>\n",
       "      <td>0.011034</td>\n",
       "      <td>0.003397</td>\n",
       "      <td>0.002577</td>\n",
       "      <td>0.004634</td>\n",
       "      <td>0.000412</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000755</td>\n",
       "      <td>0.002163</td>\n",
       "      <td>0.003602</td>\n",
       "      <td>0.023865</td>\n",
       "      <td>0.008360</td>\n",
       "      <td>0.000626</td>\n",
       "      <td>0.002521</td>\n",
       "      <td>0.002042</td>\n",
       "      <td>0.000779</td>\n",
       "      <td>0.001769</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>id_0027f1083</td>\n",
       "      <td>0.001410</td>\n",
       "      <td>0.001233</td>\n",
       "      <td>0.001858</td>\n",
       "      <td>0.015506</td>\n",
       "      <td>0.019949</td>\n",
       "      <td>0.003358</td>\n",
       "      <td>0.005273</td>\n",
       "      <td>0.002509</td>\n",
       "      <td>0.000525</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000879</td>\n",
       "      <td>0.000934</td>\n",
       "      <td>0.003337</td>\n",
       "      <td>0.001541</td>\n",
       "      <td>0.001119</td>\n",
       "      <td>0.000767</td>\n",
       "      <td>0.001287</td>\n",
       "      <td>0.001722</td>\n",
       "      <td>0.000445</td>\n",
       "      <td>0.001518</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 207 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "         sig_id  5-alpha_reductase_inhibitor  11-beta-hsd1_inhibitor  \\\n",
       "0  id_0004d9e33                     0.000750                0.001091   \n",
       "1  id_001897cda                     0.000415                0.000833   \n",
       "2  id_002429b5b                     0.000000                0.000000   \n",
       "3  id_00276f245                     0.000664                0.000753   \n",
       "4  id_0027f1083                     0.001410                0.001233   \n",
       "\n",
       "   acat_inhibitor  acetylcholine_receptor_agonist  \\\n",
       "0        0.002089                        0.019748   \n",
       "1        0.001801                        0.002423   \n",
       "2        0.000000                        0.000000   \n",
       "3        0.002374                        0.009352   \n",
       "4        0.001858                        0.015506   \n",
       "\n",
       "   acetylcholine_receptor_antagonist  acetylcholinesterase_inhibitor  \\\n",
       "0                           0.018584                        0.004491   \n",
       "1                           0.001488                        0.001938   \n",
       "2                           0.000000                        0.000000   \n",
       "3                           0.011034                        0.003397   \n",
       "4                           0.019949                        0.003358   \n",
       "\n",
       "   adenosine_receptor_agonist  adenosine_receptor_antagonist  \\\n",
       "0                    0.002085                       0.005966   \n",
       "1                    0.002919                       0.012416   \n",
       "2                    0.000000                       0.000000   \n",
       "3                    0.002577                       0.004634   \n",
       "4                    0.005273                       0.002509   \n",
       "\n",
       "   adenylyl_cyclase_activator  ...  tropomyosin_receptor_kinase_inhibitor  \\\n",
       "0                    0.000325  ...                               0.001072   \n",
       "1                    0.006070  ...                               0.000697   \n",
       "2                    0.000000  ...                               0.000000   \n",
       "3                    0.000412  ...                               0.000755   \n",
       "4                    0.000525  ...                               0.000879   \n",
       "\n",
       "   trpv_agonist  trpv_antagonist  tubulin_inhibitor  \\\n",
       "0      0.001195         0.004213           0.001195   \n",
       "1      0.001371         0.003716           0.000455   \n",
       "2      0.000000         0.000000           0.000000   \n",
       "3      0.002163         0.003602           0.023865   \n",
       "4      0.000934         0.003337           0.001541   \n",
       "\n",
       "   tyrosine_kinase_inhibitor  ubiquitin_specific_protease_inhibitor  \\\n",
       "0                   0.000634                               0.000711   \n",
       "1                   0.010757                               0.000635   \n",
       "2                   0.000000                               0.000000   \n",
       "3                   0.008360                               0.000626   \n",
       "4                   0.001119                               0.000767   \n",
       "\n",
       "   vegfr_inhibitor  vitamin_b  vitamin_d_receptor_agonist  wnt_inhibitor  \n",
       "0         0.000899   0.001795                    0.005004       0.001838  \n",
       "1         0.020449   0.001069                    0.005172       0.003133  \n",
       "2         0.000000   0.000000                    0.000000       0.000000  \n",
       "3         0.002521   0.002042                    0.000779       0.001769  \n",
       "4         0.001287   0.001722                    0.000445       0.001518  \n",
       "\n",
       "[5 rows x 207 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:10:05.753112Z",
     "iopub.status.busy": "2020-11-26T05:10:05.751602Z",
     "iopub.status.idle": "2020-11-26T05:10:05.949185Z",
     "shell.execute_reply": "2020-11-26T05:10:05.949860Z"
    },
    "papermill": {
     "duration": 8.836746,
     "end_time": "2020-11-26T05:10:05.950025",
     "exception": false,
     "start_time": "2020-11-26T05:09:57.113279",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0       0.729914\n",
       "1       0.676417\n",
       "2       0.000000\n",
       "3       0.687571\n",
       "4       0.638297\n",
       "          ...   \n",
       "3977    0.740591\n",
       "3978    0.675896\n",
       "3979    0.693348\n",
       "3980    0.711731\n",
       "3981    0.671265\n",
       "Length: 3982, dtype: float64"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "submission.sum(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2020-11-26T05:10:23.758928Z",
     "iopub.status.busy": "2020-11-26T05:10:23.757932Z",
     "iopub.status.idle": "2020-11-26T05:10:26.459338Z",
     "shell.execute_reply": "2020-11-26T05:10:26.458125Z"
    },
    "papermill": {
     "duration": 11.668806,
     "end_time": "2020-11-26T05:10:26.459480",
     "exception": false,
     "start_time": "2020-11-26T05:10:14.790674",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "submission.to_csv('submission.csv',index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "papermill": {
     "duration": 8.918951,
     "end_time": "2020-11-26T05:10:45.104409",
     "exception": false,
     "start_time": "2020-11-26T05:10:36.185458",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  },
  "papermill": {
   "duration": 7548.89905,
   "end_time": "2020-11-26T05:10:55.870812",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2020-11-26T03:05:06.971762",
   "version": "2.1.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}