OpenSourse Sample MIL warwick_Experiment 384 401 lf05 p32 MIL Train.ipynb

{"cells":[{"cell_type":"markdown","metadata":{"id":"-1_HUut4YYm5"},"source":["## This is the official training script of MIL tasks\n","* Use google colab pro+ (high RAM+GPU)\n","* we use the P100 GPU for the Experiments\n","\n","## The code and Training process along with all record are private\n","* Our github page: https://github.com/sagizty/MIL-SI\n","* The ROSE dataset is not publicly aviliable.\n","* However the MICCAI 2015 chanllenge dataset is avaliable for illustration.\n"]},{"cell_type":"code","execution_count":1,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":18,"status":"ok","timestamp":1660558725470,"user":{"displayName":"Rush MSHT","userId":"17678018738220178064"},"user_tz":-480},"id":"ZnbrNSoSXFm5","outputId":"c0d31cbf-7e26-4f07-f1ff-a6a36ee9cd1a"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mon Aug 15 10:18:44 2022       \n","+-----------------------------------------------------------------------------+\n","| NVIDIA-SMI 460.32.03    Driver Version: 460.32.03    CUDA Version: 11.2     |\n","|-------------------------------+----------------------+----------------------+\n","| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n","| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n","|                               |                      |               MIG M. |\n","|===============================+======================+======================|\n","|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n","| N/A   47C    P0    27W / 250W |      0MiB / 16280MiB |      0%      Default |\n","|                               |                      |                  N/A |\n","+-------------------------------+----------------------+----------------------+\n","                                                                               \n","+-----------------------------------------------------------------------------+\n","| Processes:                                                                  |\n","|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n","|        ID   ID                                                   Usage      |\n","|=============================================================================|\n","|  No running processes found                                                 |\n","+-----------------------------------------------------------------------------+\n"]}],"source":["# check GPU\n","!nvidia-smi"]},{"cell_type":"code","execution_count":2,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":13,"status":"ok","timestamp":1660558725471,"user":{"displayName":"Rush MSHT","userId":"17678018738220178064"},"user_tz":-480},"id":"n9GPOn5gcykA","outputId":"0e13e1a2-2f3f-4f63-bc88-f79b20a3e624"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mon Aug 15 18:18:44 UTC 2022\n"]}],"source":["!date --date='+8 hour'  # CST time zone"]},{"cell_type":"markdown","metadata":{"id":"fbnpeHYUgsJz"},"source":["## Mount Google Drive"]},{"cell_type":"code","execution_count":3,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"executionInfo":{"elapsed":20782,"status":"ok","timestamp":1660558746247,"user":{"displayName":"Rush MSHT","userId":"17678018738220178064"},"user_tz":-480},"id":"3obRNrIaffjK","outputId":"94f13204-f402-47fb-c5d7-d4e98c248c40"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mounted at /content/drive\n"]}],"source":["from google.colab import drive\n","drive.mount('/content/drive')"]},{"cell_type":"markdown","metadata":{"id":"BYevYeMFYmlx"},"source":["## create file-system enviroment\n","* mount your google drive first\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true,"base_uri":"https://localhost:8080/"},"id":"ePtQFcQCEPlu"},"outputs":[{"name":"stdout","output_type":"stream","text":["Folder Tree Creation completed!\n","Cloning into '/home/MIL_Experiment/code'...\n","warning: redirecting to https://github.com/sagizty/MIL-SI.git/\n","remote: Enumerating objects: 841, done.\u001b[K\n","remote: Counting objects: 100% (43/43), done.\u001b[K\n","remote: Compressing objects: 100% (43/43), done.\u001b[K\n","remote: Total 841 (delta 18), reused 0 (delta 0), pack-reused 798\u001b[K\n","Receiving objects: 100% (841/841), 627.71 MiB | 29.71 MiB/s, done.\n","Resolving deltas: 100% (145/145), done.\n","Checking out files: 100% (622/622), done.\n","code transfer from github completed!\n","data transfer completed!\n"]}],"source":["# create file-system enviroment\n","# mount the google drive first\n","# https://drive.google.com/drive/u/1/my-drive\n","\n","# clear colab path\n","!rm -rf /data\n","!rm -rf /home/MIL_Experiment\n","\n","# create path\n","!mkdir /home/MIL_Experiment\n","!mkdir /home/MIL_Experiment/runs\n","!mkdir /home/MIL_Experiment/code\n","!mkdir /home/MIL_Experiment/saved_models\n","!mkdir /home/MIL_Experiment/imaging_results\n","\n","!mkdir /data\n","!mkdir /data/MIL_Experiment\n","!mkdir /data/MIL_Experiment/dataset\n","\n","print('Folder Tree Creation completed!')\n","\n","# get the latest code from Github MIL-SI official page\n","!git clone https://www.github.com/sagizty/MIL-SI.git /home/MIL_Experiment/code\n","print('code transfer from github completed!')\n","\n","# copy runs if u want to compare\n","# !cp -r /content/drive/MyDrive/MIL_Experiment/runs/* /home/MIL_Experiment/runs\n","# print('tensorboard log transfer completed!')\n","\n","# copy saved_models if u want to compare\n","# !cp -r /content/drive/MyDrive/MIL_Experiment/saved_models/* /home/MIL_Experiment/saved_models\n","# print('saved_models transfer completed!')\n","\n","# get the MIL and CLS dataset from github\n","# by its zip\n","!cp /home/MIL_Experiment/code/sample_datasets/warwick_MIL.zip /data/MIL_Experiment/dataset/\n","!cp /home/MIL_Experiment/code/sample_datasets/warwick_CLS.zip /data/MIL_Experiment/dataset/\n","# unzip\n","!unzip -q /data/MIL_Experiment/dataset/warwick_MIL.zip -d /data/MIL_Experiment/dataset/\n","!unzip -q /data/MIL_Experiment/dataset/warwick_CLS.zip -d /data/MIL_Experiment/dataset/\n","# alter the path\n","!rm -rf /data/MIL_Experiment/dataset/warwick_MIL.zip\n","!rm -rf /data/MIL_Experiment/dataset/warwick_CLS.zip\n","print('data transfer completed!')"]},{"cell_type":"markdown","metadata":{"id":"xLxxHGq_wwwL"},"source":["## Arrange the working enviorment"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"K1Yb2b6TGF4r"},"outputs":[{"name":"stdout","output_type":"stream","text":["/home/MIL_Experiment/code\n","\u001b[K     |████████████▌                   | 834.1 MB 69.0 MB/s eta 0:00:19tcmalloc: large alloc 1147494400 bytes == 0x390f6000 @  0x7f95db241615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n","\u001b[K     |███████████████▉                | 1055.7 MB 1.4 MB/s eta 0:13:22tcmalloc: large alloc 1434370048 bytes == 0x7d74c000 @  0x7f95db241615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n","\u001b[K     |████████████████████            | 1336.2 MB 77.7 MB/s eta 0:00:11tcmalloc: large alloc 1792966656 bytes == 0x257e000 @  0x7f95db241615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n","\u001b[K     |█████████████████████████▎      | 1691.1 MB 1.2 MB/s eta 0:05:59tcmalloc: large alloc 2241208320 bytes == 0x6d366000 @  0x7f95db241615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x548ae9 0x51566f 0x549576 0x593fce 0x548ae9 0x5127f1 0x598e3b 0x511f68 0x598e3b 0x511f68 0x598e3b 0x511f68 0x4bc98a 0x532e76 0x594b72 0x515600 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x5118f8 0x593dd7\n","\u001b[K     |████████████████████████████████| 2137.6 MB 1.2 MB/s eta 0:00:01tcmalloc: large alloc 2137645056 bytes == 0xf2cc8000 @  0x7f95db2401e7 0x4a3940 0x4a39cc 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x593dd7 0x511e2c 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x548ae9\n","tcmalloc: large alloc 2672058368 bytes == 0x1e6822000 @  0x7f95db241615 0x592b76 0x4df71e 0x59afff 0x515655 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x549576 0x593fce 0x511e2c 0x593dd7 0x511e2c 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576 0x593fce 0x548ae9 0x5127f1 0x549576\n","\u001b[K     |████████████████████████████████| 2137.6 MB 383 bytes/s \n","\u001b[K     |████████████████████████████████| 24.5 MB 1.2 MB/s \n","\u001b[31mERROR: pip's dependency resolver does not currently take into account all the packages that are installed. This behaviour is the source of the following dependency conflicts.\n","torchtext 0.13.1 requires torch==1.12.1, but you have torch 1.10.0+cu111 which is incompatible.\n","torchaudio 0.12.1+cu113 requires torch==1.12.1, but you have torch 1.10.0+cu111 which is incompatible.\u001b[0m\n","\u001b[?25hLooking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting tensorboardX\n","  Downloading tensorboardX-2.5.1-py2.py3-none-any.whl (125 kB)\n","\u001b[K     |████████████████████████████████| 125 kB 12.6 MB/s \n","\u001b[?25hRequirement already satisfied: protobuf\u003c=3.20.1,\u003e=3.8.0 in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (3.17.3)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from tensorboardX) (1.21.6)\n","Requirement already satisfied: six\u003e=1.9 in /usr/local/lib/python3.7/dist-packages (from protobuf\u003c=3.20.1,\u003e=3.8.0-\u003etensorboardX) (1.15.0)\n","Installing collected packages: tensorboardX\n","Successfully installed tensorboardX-2.5.1\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting timm==0.5.4\n","  Downloading timm-0.5.4-py3-none-any.whl (431 kB)\n","\u001b[K     |████████████████████████████████| 431 kB 12.1 MB/s \n","\u001b[?25hRequirement already satisfied: torch\u003e=1.4 in /usr/local/lib/python3.7/dist-packages (from timm==0.5.4) (1.10.0+cu111)\n","Requirement already satisfied: torchvision in /usr/local/lib/python3.7/dist-packages (from timm==0.5.4) (0.11.1+cu111)\n","Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch\u003e=1.4-\u003etimm==0.5.4) (4.1.1)\n","Requirement already satisfied: pillow!=8.3.0,\u003e=5.3.0 in /usr/local/lib/python3.7/dist-packages (from torchvision-\u003etimm==0.5.4) (7.1.2)\n","Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torchvision-\u003etimm==0.5.4) (1.21.6)\n","Installing collected packages: timm\n","Successfully installed timm-0.5.4\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting notifyemail\n","  Downloading notifyemail-1.0.2-py3-none-any.whl (31 kB)\n","Installing collected packages: notifyemail\n","Successfully installed notifyemail-1.0.2\n","Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n","Collecting ttach\n","  Downloading ttach-0.0.3-py3-none-any.whl (9.8 kB)\n","Installing collected packages: ttach\n","Successfully installed ttach-0.0.3\n"]}],"source":["# change working dir\n","import os\n","os.chdir(\"/home/MIL_Experiment/code\")\n","!pwd\n","\n","# get packages\n","!pip install -q torch==1.10.0+cu111 torchvision==0.11.1+cu111 -f https://download.pytorch.org/whl/torch_stable.html\n","!pip install tensorboardX\n","!pip install timm==0.5.4\n","!pip install notifyemail\n","!pip install ttach\n"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"87Owjg_pN2yD"},"outputs":[{"name":"stdout","output_type":"stream","text":["Python 3.7.13\n"]}],"source":["!python --version"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"GpEVUWwqK79D"},"outputs":[{"name":"stdout","output_type":"stream","text":["Package                       Version\n","----------------------------- ----------------------------\n","absl-py                       1.2.0\n","aiohttp                       3.8.1\n","aiosignal                     1.2.0\n","alabaster                     0.7.12\n","albumentations                1.2.1\n","altair                        4.2.0\n","appdirs                       1.4.4\n","argon2-cffi                   21.3.0\n","argon2-cffi-bindings          21.2.0\n","arviz                         0.12.1\n","astor                         0.8.1\n","astropy                       4.3.1\n","astunparse                    1.6.3\n","async-timeout                 4.0.2\n","asynctest                     0.13.0\n","atari-py                      0.2.9\n","atomicwrites                  1.4.1\n","attrs                         22.1.0\n","audioread                     2.1.9\n","autograd                      1.4\n","Babel                         2.10.3\n","backcall                      0.2.0\n","beautifulsoup4                4.6.3\n","bleach                        5.0.1\n","blis                          0.7.8\n","bokeh                         2.3.3\n","branca                        0.5.0\n","bs4                           0.0.1\n","CacheControl                  0.12.11\n","cached-property               1.5.2\n","cachetools                    4.2.4\n","catalogue                     2.0.8\n","certifi                       2022.6.15\n","cffi                          1.15.1\n","cftime                        1.6.1\n","chardet                       3.0.4\n","charset-normalizer            2.1.0\n","click                         7.1.2\n","clikit                        0.6.2\n","cloudpickle                   1.3.0\n","cmake                         3.22.6\n","cmdstanpy                     1.0.4\n","colorcet                      3.0.0\n","colorlover                    0.3.0\n","community                     1.0.0b1\n","contextlib2                   0.5.5\n","convertdate                   2.4.0\n","crashtest                     0.3.1\n","crcmod                        1.7\n","cufflinks                     0.17.3\n","cupy-cuda111                  9.4.0\n","cvxopt                        1.3.0\n","cvxpy                         1.2.1\n","cycler                        0.11.0\n","cymem                         2.0.6\n","Cython                        0.29.32\n","daft                          0.0.4\n","dask                          2.12.0\n","datascience                   0.17.5\n","debugpy                       1.0.0\n","decorator                     4.4.2\n","defusedxml                    0.7.1\n","deprecat                      2.1.1\n","descartes                     1.1.0\n","dill                          0.3.5.1\n","distributed                   1.25.3\n","dlib                          19.24.0\n","dm-tree                       0.1.7\n","docutils                      0.17.1\n","dopamine-rl                   1.0.5\n","earthengine-api               0.1.318\n","easydict                      1.9\n","ecos                          2.0.10\n","editdistance                  0.5.3\n","en-core-web-sm                3.4.0\n","entrypoints                   0.4\n","ephem                         4.1.3\n","et-xmlfile                    1.1.0\n","etils                         0.7.1\n","fa2                           0.3.5\n","fastai                        2.7.9\n","fastcore                      1.5.16\n","fastdownload                  0.0.7\n","fastdtw                       0.3.4\n","fastjsonschema                2.16.1\n","fastprogress                  1.0.3\n","fastrlock                     0.8\n","feather-format                0.4.1\n","filelock                      3.7.1\n","firebase-admin                4.4.0\n","fix-yahoo-finance             0.0.22\n","Flask                         1.1.4\n","flatbuffers                   2.0\n","folium                        0.12.1.post1\n","frozenlist                    1.3.1\n","future                        0.16.0\n","gast                          0.5.3\n","GDAL                          2.2.2\n","gdown                         4.4.0\n","gensim                        3.6.0\n","geographiclib                 1.52\n","geopy                         1.17.0\n","gin-config                    0.5.0\n","glob2                         0.7\n","google                        2.0.3\n","google-api-core               1.31.6\n","google-api-python-client      1.12.11\n","google-auth                   1.35.0\n","google-auth-httplib2          0.0.4\n","google-auth-oauthlib          0.4.6\n","google-cloud-bigquery         1.21.0\n","google-cloud-bigquery-storage 1.1.2\n","google-cloud-core             1.0.3\n","google-cloud-datastore        1.8.0\n","google-cloud-firestore        1.7.0\n","google-cloud-language         1.2.0\n","google-cloud-storage          1.18.1\n","google-cloud-translate        1.5.0\n","google-colab                  1.0.0\n","google-pasta                  0.2.0\n","google-resumable-media        0.4.1\n","googleapis-common-protos      1.56.4\n","googledrivedownloader         0.4\n","graphviz                      0.10.1\n","greenlet                      1.1.2\n","grpcio                        1.47.0\n","gspread                       3.4.2\n","gspread-dataframe             3.0.8\n","gym                           0.17.3\n","h5py                          3.1.0\n","HeapDict                      1.0.1\n","hijri-converter               2.2.4\n","holidays                      0.14.2\n","holoviews                     1.14.9\n","html5lib                      1.0.1\n","httpimport                    0.5.18\n","httplib2                      0.17.4\n","httplib2shim                  0.0.3\n","httpstan                      4.6.1\n","humanize                      0.5.1\n","hyperopt                      0.1.2\n","idna                          2.10\n","imageio                       2.9.0\n","imagesize                     1.4.1\n","imbalanced-learn              0.8.1\n","imblearn                      0.0\n","imgaug                        0.4.0\n","importlib-metadata            4.12.0\n","importlib-resources           5.9.0\n","imutils                       0.5.4\n","inflect                       2.1.0\n","intel-openmp                  2022.1.0\n","intervaltree                  2.1.0\n","ipykernel                     4.10.1\n","ipython                       5.5.0\n","ipython-genutils              0.2.0\n","ipython-sql                   0.3.9\n","ipywidgets                    7.7.1\n","itsdangerous                  1.1.0\n","jax                           0.3.14\n","jaxlib                        0.3.14+cuda11.cudnn805\n","jedi                          0.18.1\n","jieba                         0.42.1\n","Jinja2                        2.11.3\n","joblib                        1.1.0\n","jpeg4py                       0.1.4\n","jsonschema                    4.3.3\n","jupyter                       1.0.0\n","jupyter-client                5.3.5\n","jupyter-console               5.2.0\n","jupyter-core                  4.11.1\n","jupyterlab-pygments           0.2.2\n","jupyterlab-widgets            1.1.1\n","kaggle                        1.5.12\n","kapre                         0.3.7\n","keras                         2.8.0\n","Keras-Preprocessing           1.1.2\n","keras-vis                     0.4.1\n","kiwisolver                    1.4.4\n","korean-lunar-calendar         0.2.1\n","langcodes                     3.3.0\n","libclang                      14.0.6\n","librosa                       0.8.1\n","lightgbm                      2.2.3\n","llvmlite                      0.39.0\n","lmdb                          0.99\n","LunarCalendar                 0.0.9\n","lxml                          4.9.1\n","Markdown                      3.4.1\n","MarkupSafe                    2.0.1\n","marshmallow                   3.17.0\n","matplotlib                    3.2.2\n","matplotlib-inline             0.1.3\n","matplotlib-venn               0.11.7\n","missingno                     0.5.1\n","mistune                       0.8.4\n","mizani                        0.6.0\n","mkl                           2019.0\n","mlxtend                       0.14.0\n","more-itertools                8.14.0\n","moviepy                       0.2.3.5\n","mpmath                        1.2.1\n","msgpack                       1.0.4\n","multidict                     6.0.2\n","multitasking                  0.0.11\n","murmurhash                    1.0.7\n","music21                       5.5.0\n","natsort                       5.5.0\n","nbclient                      0.6.6\n","nbconvert                     5.6.1\n","nbformat                      5.4.0\n","nest-asyncio                  1.5.5\n","netCDF4                       1.6.0\n","networkx                      2.6.3\n","nibabel                       3.0.2\n","nltk                          3.7\n","notebook                      5.3.1\n","notifyemail                   1.0.2\n","numba                         0.56.0\n","numexpr                       2.8.3\n","numpy                         1.21.6\n","oauth2client                  4.1.3\n","oauthlib                      3.2.0\n","okgrade                       0.4.3\n","opencv-contrib-python         4.6.0.66\n","opencv-python                 4.6.0.66\n","opencv-python-headless        4.6.0.66\n","openpyxl                      3.0.10\n","opt-einsum                    3.3.0\n","osqp                          0.6.2.post0\n","packaging                     21.3\n","palettable                    3.3.0\n","pandas                        1.3.5\n","pandas-datareader             0.9.0\n","pandas-gbq                    0.13.3\n","pandas-profiling              1.4.1\n","pandocfilters                 1.5.0\n","panel                         0.12.1\n","param                         1.12.2\n","parso                         0.8.3\n","pastel                        0.2.1\n","pathlib                       1.0.1\n","pathy                         0.6.2\n","patsy                         0.5.2\n","pep517                        0.13.0\n","pexpect                       4.8.0\n","pickleshare                   0.7.5\n","Pillow                        7.1.2\n","pip                           21.1.3\n","pip-tools                     6.2.0\n","pkgutil-resolve-name          1.3.10\n","plotly                        5.5.0\n","plotnine                      0.6.0\n","pluggy                        0.7.1\n","pooch                         1.6.0\n","portpicker                    1.3.9\n","prefetch-generator            1.0.1\n","preshed                       3.0.6\n","prettytable                   3.3.0\n","progressbar2                  3.38.0\n","prometheus-client             0.14.1\n","promise                       2.3\n","prompt-toolkit                1.0.18\n","prophet                       1.1\n","protobuf                      3.17.3\n","psutil                        5.4.8\n","psycopg2                      2.9.3\n","ptyprocess                    0.7.0\n","py                            1.11.0\n","pyarrow                       6.0.1\n","pyasn1                        0.4.8\n","pyasn1-modules                0.2.8\n","pycocotools                   2.0.4\n","pycparser                     2.21\n","pyct                          0.4.8\n","pydantic                      1.9.1\n","pydata-google-auth            1.4.0\n","pydot                         1.3.0\n","pydot-ng                      2.0.0\n","pydotplus                     2.0.2\n","PyDrive                       1.3.1\n","pyemd                         0.5.1\n","pyerfa                        2.0.0.1\n","pyglet                        1.5.0\n","Pygments                      2.6.1\n","pygobject                     3.26.1\n","pylev                         1.4.0\n","pymc3                         3.11.5\n","PyMeeus                       0.5.11\n","pymongo                       4.2.0\n","pymystem3                     0.2.0\n","PyOpenGL                      3.1.6\n","pyparsing                     3.0.9\n","pyrsistent                    0.18.1\n","pysimdjson                    3.2.0\n","pysndfile                     1.3.8\n","PySocks                       1.7.1\n","pystan                        3.3.0\n","pytest                        3.6.4\n","python-apt                    0.0.0\n","python-chess                  0.23.11\n","python-dateutil               2.8.2\n","python-louvain                0.16\n","python-slugify                6.1.2\n","python-utils                  3.3.3\n","pytz                          2022.1\n","pyviz-comms                   2.2.0\n","PyWavelets                    1.3.0\n","PyYAML                        3.13\n","pyzmq                         23.2.0\n","qdldl                         0.1.5.post2\n","qtconsole                     5.3.1\n","QtPy                          2.2.0\n","qudida                        0.0.4\n","regex                         2022.6.2\n","requests                      2.23.0\n","requests-oauthlib             1.3.1\n","resampy                       0.4.0\n","rpy2                          3.4.5\n","rsa                           4.9\n","scikit-image                  0.18.3\n","scikit-learn                  1.0.2\n","scipy                         1.7.3\n","screen-resolution-extra       0.0.0\n","scs                           3.2.0\n","seaborn                       0.11.2\n","semver                        2.13.0\n","Send2Trash                    1.8.0\n","setuptools                    57.4.0\n","setuptools-git                1.2\n","Shapely                       1.8.2\n","simplegeneric                 0.8.1\n","six                           1.15.0\n","sklearn                       0.0\n","sklearn-pandas                1.8.0\n","smart-open                    5.2.1\n","snowballstemmer               2.2.0\n","sortedcontainers              2.4.0\n","SoundFile                     0.10.3.post1\n","soupsieve                     2.3.2.post1\n","spacy                         3.4.1\n","spacy-legacy                  3.0.9\n","spacy-loggers                 1.0.3\n","Sphinx                        1.8.6\n","sphinxcontrib-serializinghtml 1.1.5\n","sphinxcontrib-websupport      1.2.4\n","SQLAlchemy                    1.4.40\n","sqlparse                      0.4.2\n","srsly                         2.4.4\n","statsmodels                   0.10.2\n","sympy                         1.7.1\n","tables                        3.7.0\n","tabulate                      0.8.10\n","tblib                         1.7.0\n","tenacity                      8.0.1\n","tensorboard                   2.8.0\n","tensorboard-data-server       0.6.1\n","tensorboard-plugin-wit        1.8.1\n","tensorboardX                  2.5.1\n","tensorflow                    2.8.2+zzzcolab20220719082949\n","tensorflow-datasets           4.6.0\n","tensorflow-estimator          2.8.0\n","tensorflow-gcs-config         2.8.0\n","tensorflow-hub                0.12.0\n","tensorflow-io-gcs-filesystem  0.26.0\n","tensorflow-metadata           1.9.0\n","tensorflow-probability        0.16.0\n","termcolor                     1.1.0\n","terminado                     0.13.3\n","testpath                      0.6.0\n","text-unidecode                1.3\n","textblob                      0.15.3\n","Theano-PyMC                   1.1.2\n","thinc                         8.1.0\n","threadpoolctl                 3.1.0\n","tifffile                      2021.11.2\n","timm                          0.5.4\n","tinycss2                      1.1.1\n","toml                          0.10.2\n","tomli                         2.0.1\n","toolz                         0.12.0\n","torch                         1.10.0+cu111\n","torchaudio                    0.12.1+cu113\n","torchsummary                  1.5.1\n","torchtext                     0.13.1\n","torchvision                   0.11.1+cu111\n","tornado                       5.1.1\n","tqdm                          4.64.0\n","traitlets                     5.1.1\n","ttach                         0.0.3\n","tweepy                        3.10.0\n","typeguard                     2.7.1\n","typer                         0.4.2\n","typing-extensions             4.1.1\n","tzlocal                       1.5.1\n","ujson                         5.4.0\n","uritemplate                   3.0.1\n","urllib3                       1.24.3\n","vega-datasets                 0.9.0\n","wasabi                        0.10.1\n","wcwidth                       0.2.5\n","webargs                       8.2.0\n","webencodings                  0.5.1\n","Werkzeug                      1.0.1\n","wheel                         0.37.1\n","widgetsnbextension            3.6.1\n","wordcloud                     1.8.2.2\n","wrapt                         1.14.1\n","xarray                        0.20.2\n","xarray-einstats               0.2.2\n","xgboost                       0.90\n","xkit                          0.0.0\n","xlrd                          1.1.0\n","xlwt                          1.3.0\n","yarl                          1.8.1\n","yellowbrick                   1.4\n","zict                          2.2.0\n","zipp                          3.8.1\n"]}],"source":["!pip list"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"4IBN4wS4HiM_"},"outputs":[],"source":["!pip freeze\u003erequirements.txt\n","!cp requirements.txt ../runs"]},{"cell_type":"markdown","metadata":{"id":"h31KAx1ZZEl9"},"source":["## Start Training\n","* by command line\n","* use argparse to set down hyper-parameter\n","\n","* 5-fold experiment is used here"]},{"cell_type":"markdown","metadata":{"id":"ruFuG07xOftS"},"source":["# MIL Experiments"]},{"cell_type":"markdown","metadata":{"id":"ZMeCgHQ7OpFQ"},"source":["* Train"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"bnCOfr2pXjs4"},"outputs":[{"name":"stdout","output_type":"stream","text":["GPU: 0\n","*********************************setting*************************************\n","Namespace(CLS_MIL_head_weight=1.0, CLS_MIL_off=False, MIL_head_weight=1.0, Pre_Trained_model_path=None, backbone_PT_off=False, batch_size=4, check_minibatch=5, data_augmentation_mode=2, dataroot='/data/MIL_Experiment/dataset/warwick_MIL', draw_root='/home/MIL_Experiment/runs', edge_size=384, enable_attention_check=False, enable_notify=False, enable_sam=False, enable_tensorboard=True, enable_visualize_check=False, gpu_idx=0, intake_epochs=0, lr=1e-05, lrf=0.05, model_idx='ViT_384_401_PT_lf05_b4_p32_warwick_MIL', model_path='/home/MIL_Experiment/saved_models', num_epochs=50, num_workers=2, opt_name='Adam', paint=True, patch_size=32, shuffle_MIL_off=False)\n","----------------------------------------------------------------\n","        Layer (type)               Output Shape         Param #\n","================================================================\n","            Conv2d-1          [-1, 768, 24, 24]         590,592\n","        PatchEmbed-2             [-1, 576, 768]               0\n","           Dropout-3             [-1, 577, 768]               0\n","         LayerNorm-4             [-1, 577, 768]           1,536\n","            Linear-5            [-1, 577, 2304]       1,771,776\n","           Dropout-6         [-1, 12, 577, 577]               0\n","            Linear-7             [-1, 577, 768]         590,592\n","           Dropout-8             [-1, 577, 768]               0\n","         Attention-9             [-1, 577, 768]               0\n","         Identity-10             [-1, 577, 768]               0\n","        LayerNorm-11             [-1, 577, 768]           1,536\n","           Linear-12            [-1, 577, 3072]       2,362,368\n","             GELU-13            [-1, 577, 3072]               0\n","          Dropout-14            [-1, 577, 3072]               0\n","           Linear-15             [-1, 577, 768]       2,360,064\n","          Dropout-16             [-1, 577, 768]               0\n","              FFN-17             [-1, 577, 768]               0\n","         Identity-18             [-1, 577, 768]               0\n","    Encoder_Block-19             [-1, 577, 768]               0\n","        LayerNorm-20             [-1, 577, 768]           1,536\n","           Linear-21            [-1, 577, 2304]       1,771,776\n","          Dropout-22         [-1, 12, 577, 577]               0\n","           Linear-23             [-1, 577, 768]         590,592\n","          Dropout-24             [-1, 577, 768]               0\n","        Attention-25             [-1, 577, 768]               0\n","         Identity-26             [-1, 577, 768]               0\n","        LayerNorm-27             [-1, 577, 768]           1,536\n","           Linear-28            [-1, 577, 3072]       2,362,368\n","             GELU-29            [-1, 577, 3072]               0\n","          Dropout-30            [-1, 577, 3072]               0\n","           Linear-31             [-1, 577, 768]       2,360,064\n","          Dropout-32             [-1, 577, 768]               0\n","              FFN-33             [-1, 577, 768]               0\n","         Identity-34             [-1, 577, 768]               0\n","    Encoder_Block-35             [-1, 577, 768]               0\n","        LayerNorm-36             [-1, 577, 768]           1,536\n","           Linear-37            [-1, 577, 2304]       1,771,776\n","          Dropout-38         [-1, 12, 577, 577]               0\n","           Linear-39             [-1, 577, 768]         590,592\n","          Dropout-40             [-1, 577, 768]               0\n","        Attention-41             [-1, 577, 768]               0\n","         Identity-42             [-1, 577, 768]               0\n","        LayerNorm-43             [-1, 577, 768]           1,536\n","           Linear-44            [-1, 577, 3072]       2,362,368\n","             GELU-45            [-1, 577, 3072]               0\n","          Dropout-46            [-1, 577, 3072]               0\n","           Linear-47             [-1, 577, 768]       2,360,064\n","          Dropout-48             [-1, 577, 768]               0\n","              FFN-49             [-1, 577, 768]               0\n","         Identity-50             [-1, 577, 768]               0\n","    Encoder_Block-51             [-1, 577, 768]               0\n","        LayerNorm-52             [-1, 577, 768]           1,536\n","           Linear-53            [-1, 577, 2304]       1,771,776\n","          Dropout-54         [-1, 12, 577, 577]               0\n","           Linear-55             [-1, 577, 768]         590,592\n","          Dropout-56             [-1, 577, 768]               0\n","        Attention-57             [-1, 577, 768]               0\n","         Identity-58             [-1, 577, 768]               0\n","        LayerNorm-59             [-1, 577, 768]           1,536\n","           Linear-60            [-1, 577, 3072]       2,362,368\n","             GELU-61            [-1, 577, 3072]               0\n","          Dropout-62            [-1, 577, 3072]               0\n","           Linear-63             [-1, 577, 768]       2,360,064\n","          Dropout-64             [-1, 577, 768]               0\n","              FFN-65             [-1, 577, 768]               0\n","         Identity-66             [-1, 577, 768]               0\n","    Encoder_Block-67             [-1, 577, 768]               0\n","        LayerNorm-68             [-1, 577, 768]           1,536\n","           Linear-69            [-1, 577, 2304]       1,771,776\n","          Dropout-70         [-1, 12, 577, 577]               0\n","           Linear-71             [-1, 577, 768]         590,592\n","          Dropout-72             [-1, 577, 768]               0\n","        Attention-73             [-1, 577, 768]               0\n","         Identity-74             [-1, 577, 768]               0\n","        LayerNorm-75             [-1, 577, 768]           1,536\n","           Linear-76            [-1, 577, 3072]       2,362,368\n","             GELU-77            [-1, 577, 3072]               0\n","          Dropout-78            [-1, 577, 3072]               0\n","           Linear-79             [-1, 577, 768]       2,360,064\n","          Dropout-80             [-1, 577, 768]               0\n","              FFN-81             [-1, 577, 768]               0\n","         Identity-82             [-1, 577, 768]               0\n","    Encoder_Block-83             [-1, 577, 768]               0\n","        LayerNorm-84             [-1, 577, 768]           1,536\n","           Linear-85            [-1, 577, 2304]       1,771,776\n","          Dropout-86         [-1, 12, 577, 577]               0\n","           Linear-87             [-1, 577, 768]         590,592\n","          Dropout-88             [-1, 577, 768]               0\n","        Attention-89             [-1, 577, 768]               0\n","         Identity-90             [-1, 577, 768]               0\n","        LayerNorm-91             [-1, 577, 768]           1,536\n","           Linear-92            [-1, 577, 3072]       2,362,368\n","             GELU-93            [-1, 577, 3072]               0\n","          Dropout-94            [-1, 577, 3072]               0\n","           Linear-95             [-1, 577, 768]       2,360,064\n","          Dropout-96             [-1, 577, 768]               0\n","              FFN-97             [-1, 577, 768]               0\n","         Identity-98             [-1, 577, 768]               0\n","    Encoder_Block-99             [-1, 577, 768]               0\n","       LayerNorm-100             [-1, 577, 768]           1,536\n","          Linear-101            [-1, 577, 2304]       1,771,776\n","         Dropout-102         [-1, 12, 577, 577]               0\n","          Linear-103             [-1, 577, 768]         590,592\n","         Dropout-104             [-1, 577, 768]               0\n","       Attention-105             [-1, 577, 768]               0\n","        Identity-106             [-1, 577, 768]               0\n","       LayerNorm-107             [-1, 577, 768]           1,536\n","          Linear-108            [-1, 577, 3072]       2,362,368\n","            GELU-109            [-1, 577, 3072]               0\n","         Dropout-110            [-1, 577, 3072]               0\n","          Linear-111             [-1, 577, 768]       2,360,064\n","         Dropout-112             [-1, 577, 768]               0\n","             FFN-113             [-1, 577, 768]               0\n","        Identity-114             [-1, 577, 768]               0\n","   Encoder_Block-115             [-1, 577, 768]               0\n","       LayerNorm-116             [-1, 577, 768]           1,536\n","          Linear-117            [-1, 577, 2304]       1,771,776\n","         Dropout-118         [-1, 12, 577, 577]               0\n","          Linear-119             [-1, 577, 768]         590,592\n","         Dropout-120             [-1, 577, 768]               0\n","       Attention-121             [-1, 577, 768]               0\n","        Identity-122             [-1, 577, 768]               0\n","       LayerNorm-123             [-1, 577, 768]           1,536\n","          Linear-124            [-1, 577, 3072]       2,362,368\n","            GELU-125            [-1, 577, 3072]               0\n","         Dropout-126            [-1, 577, 3072]               0\n","          Linear-127             [-1, 577, 768]       2,360,064\n","         Dropout-128             [-1, 577, 768]               0\n","             FFN-129             [-1, 577, 768]               0\n","        Identity-130             [-1, 577, 768]               0\n","   Encoder_Block-131             [-1, 577, 768]               0\n","       LayerNorm-132             [-1, 577, 768]           1,536\n","          Linear-133            [-1, 577, 2304]       1,771,776\n","         Dropout-134         [-1, 12, 577, 577]               0\n","          Linear-135             [-1, 577, 768]         590,592\n","         Dropout-136             [-1, 577, 768]               0\n","       Attention-137             [-1, 577, 768]               0\n","        Identity-138             [-1, 577, 768]               0\n","       LayerNorm-139             [-1, 577, 768]           1,536\n","          Linear-140            [-1, 577, 3072]       2,362,368\n","            GELU-141            [-1, 577, 3072]               0\n","         Dropout-142            [-1, 577, 3072]               0\n","          Linear-143             [-1, 577, 768]       2,360,064\n","         Dropout-144             [-1, 577, 768]               0\n","             FFN-145             [-1, 577, 768]               0\n","        Identity-146             [-1, 577, 768]               0\n","   Encoder_Block-147             [-1, 577, 768]               0\n","       LayerNorm-148             [-1, 577, 768]           1,536\n","          Linear-149            [-1, 577, 2304]       1,771,776\n","         Dropout-150         [-1, 12, 577, 577]               0\n","          Linear-151             [-1, 577, 768]         590,592\n","         Dropout-152             [-1, 577, 768]               0\n","       Attention-153             [-1, 577, 768]               0\n","        Identity-154             [-1, 577, 768]               0\n","       LayerNorm-155             [-1, 577, 768]           1,536\n","          Linear-156            [-1, 577, 3072]       2,362,368\n","            GELU-157            [-1, 577, 3072]               0\n","         Dropout-158            [-1, 577, 3072]               0\n","          Linear-159             [-1, 577, 768]       2,360,064\n","         Dropout-160             [-1, 577, 768]               0\n","             FFN-161             [-1, 577, 768]               0\n","        Identity-162             [-1, 577, 768]               0\n","   Encoder_Block-163             [-1, 577, 768]               0\n","       LayerNorm-164             [-1, 577, 768]           1,536\n","          Linear-165            [-1, 577, 2304]       1,771,776\n","         Dropout-166         [-1, 12, 577, 577]               0\n","          Linear-167             [-1, 577, 768]         590,592\n","         Dropout-168             [-1, 577, 768]               0\n","       Attention-169             [-1, 577, 768]               0\n","        Identity-170             [-1, 577, 768]               0\n","       LayerNorm-171             [-1, 577, 768]           1,536\n","          Linear-172            [-1, 577, 3072]       2,362,368\n","            GELU-173            [-1, 577, 3072]               0\n","         Dropout-174            [-1, 577, 3072]               0\n","          Linear-175             [-1, 577, 768]       2,360,064\n","         Dropout-176             [-1, 577, 768]               0\n","             FFN-177             [-1, 577, 768]               0\n","        Identity-178             [-1, 577, 768]               0\n","   Encoder_Block-179             [-1, 577, 768]               0\n","       LayerNorm-180             [-1, 577, 768]           1,536\n","          Linear-181            [-1, 577, 2304]       1,771,776\n","         Dropout-182         [-1, 12, 577, 577]               0\n","          Linear-183             [-1, 577, 768]         590,592\n","         Dropout-184             [-1, 577, 768]               0\n","       Attention-185             [-1, 577, 768]               0\n","        Identity-186             [-1, 577, 768]               0\n","       LayerNorm-187             [-1, 577, 768]           1,536\n","          Linear-188            [-1, 577, 3072]       2,362,368\n","            GELU-189            [-1, 577, 3072]               0\n","         Dropout-190            [-1, 577, 3072]               0\n","          Linear-191             [-1, 577, 768]       2,360,064\n","         Dropout-192             [-1, 577, 768]               0\n","             FFN-193             [-1, 577, 768]               0\n","        Identity-194             [-1, 577, 768]               0\n","   Encoder_Block-195             [-1, 577, 768]               0\n","       LayerNorm-196             [-1, 577, 768]           1,536\n","VisionTransformer-197             [-1, 577, 768]               0\n","          Linear-198                  [-1, 288]     127,402,272\n","            GELU-199                  [-1, 288]               0\n","          Linear-200                  [-1, 288]          83,232\n","            GELU-201                  [-1, 288]               0\n","          Linear-202                    [-1, 3]             867\n","         Dropout-203                    [-1, 3]               0\n","represtation_MLP-204                    [-1, 3]               0\n","================================================================\n","Total params: 213,132,963\n","Trainable params: 213,132,963\n","Non-trainable params: 0\n","----------------------------------------------------------------\n","Input size (MB): 1.69\n","Forward/backward pass size (MB): 1437.50\n","Params size (MB): 813.04\n","Estimated Total Size (MB): 2252.22\n","----------------------------------------------------------------\n","model : ViT_384_401_PT_lf05_b4_p32_warwick_MIL\n","Epoch 1/50\n","----------\n","Epoch: 1     train index of 5 minibatch: 1      time used: 3.734191417694092\n","minibatch AVG loss: 88.83287506103515\n","Epoch: 1     train index of 5 minibatch: 2      time used: 3.169131278991699\n","minibatch AVG loss: 66.1259552001953\n","Epoch: 1     train index of 5 minibatch: 3      time used: 3.1672613620758057\n","minibatch AVG loss: 62.41692199707031\n","\n","Epoch: 1  train \n","Loss: 69.8748  Acc: 68.1159\n","benign precision: 65.3846  recall: 56.6667\n","benign sensitivity: 56.6667  specificity: 76.9231\n","benign FPR: 23.0769  NPV: 69.7674\n","benign TP: 17.0\n","benign TN: 30.0\n","benign FP: 9.0\n","benign FN: 13.0\n","malignant precision: 69.7674  recall: 76.9231\n","malignant sensitivity: 76.9231  specificity: 56.6667\n","malignant FPR: 43.3333  NPV: 65.3846\n","malignant TP: 30.0\n","malignant TN: 17.0\n","malignant FP: 13.0\n","malignant FN: 9.0\n","\n","\n","\n","Epoch: 1  val \n","Loss: 44.9924  Acc: 87.5000\n","benign precision: 100.0000  recall: 71.4286\n","benign sensitivity: 71.4286  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 81.8182\n","benign TP: 5.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 2.0\n","malignant precision: 81.8182  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 71.4286\n","malignant FPR: 28.5714  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 5.0\n","malignant FP: 2.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 2/50\n","----------\n","Epoch: 2     train index of 5 minibatch: 1      time used: 3.6479766368865967\n","minibatch AVG loss: 48.20692672729492\n","Epoch: 2     train index of 5 minibatch: 2      time used: 3.1657590866088867\n","minibatch AVG loss: 51.977960205078126\n","Epoch: 2     train index of 5 minibatch: 3      time used: 3.1657204627990723\n","minibatch AVG loss: 50.338462829589844\n","\n","Epoch: 2  train \n","Loss: 51.3171  Acc: 85.5072\n","benign precision: 76.3158  recall: 96.6667\n","benign sensitivity: 96.6667  specificity: 76.9231\n","benign FPR: 23.0769  NPV: 96.7742\n","benign TP: 29.0\n","benign TN: 30.0\n","benign FP: 9.0\n","benign FN: 1.0\n","malignant precision: 96.7742  recall: 76.9231\n","malignant sensitivity: 76.9231  specificity: 96.6667\n","malignant FPR: 3.3333  NPV: 76.3158\n","malignant TP: 30.0\n","malignant TN: 29.0\n","malignant FP: 1.0\n","malignant FN: 9.0\n","\n","\n","\n","Epoch: 2  val \n","Loss: 47.8320  Acc: 87.5000\n","benign precision: 100.0000  recall: 71.4286\n","benign sensitivity: 71.4286  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 81.8182\n","benign TP: 5.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 2.0\n","malignant precision: 81.8182  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 71.4286\n","malignant FPR: 28.5714  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 5.0\n","malignant FP: 2.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 3/50\n","----------\n","Epoch: 3     train index of 5 minibatch: 1      time used: 3.667602300643921\n","minibatch AVG loss: 47.48723449707031\n","Epoch: 3     train index of 5 minibatch: 2      time used: 3.166125774383545\n","minibatch AVG loss: 50.27050704956055\n","Epoch: 3     train index of 5 minibatch: 3      time used: 3.1650750637054443\n","minibatch AVG loss: 45.42153205871582\n","\n","Epoch: 3  train \n","Loss: 47.9700  Acc: 85.5072\n","benign precision: 83.3333  recall: 83.3333\n","benign sensitivity: 83.3333  specificity: 87.1795\n","benign FPR: 12.8205  NPV: 87.1795\n","benign TP: 25.0\n","benign TN: 34.0\n","benign FP: 5.0\n","benign FN: 5.0\n","malignant precision: 87.1795  recall: 87.1795\n","malignant sensitivity: 87.1795  specificity: 83.3333\n","malignant FPR: 16.6667  NPV: 83.3333\n","malignant TP: 34.0\n","malignant TN: 25.0\n","malignant FP: 5.0\n","malignant FN: 5.0\n","\n","\n","\n","Epoch: 3  val \n","Loss: 37.0355  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 4/50\n","----------\n","Epoch: 4     train index of 5 minibatch: 1      time used: 3.642468214035034\n","minibatch AVG loss: 42.87559280395508\n","Epoch: 4     train index of 5 minibatch: 2      time used: 3.1620490550994873\n","minibatch AVG loss: 42.180084991455075\n","Epoch: 4     train index of 5 minibatch: 3      time used: 3.1618576049804688\n","minibatch AVG loss: 32.92283706665039\n","\n","Epoch: 4  train \n","Loss: 40.5371  Acc: 97.1014\n","benign precision: 96.6667  recall: 96.6667\n","benign sensitivity: 96.6667  specificity: 97.4359\n","benign FPR: 2.5641  NPV: 97.4359\n","benign TP: 29.0\n","benign TN: 38.0\n","benign FP: 1.0\n","benign FN: 1.0\n","malignant precision: 97.4359  recall: 97.4359\n","malignant sensitivity: 97.4359  specificity: 96.6667\n","malignant FPR: 3.3333  NPV: 96.6667\n","malignant TP: 38.0\n","malignant TN: 29.0\n","malignant FP: 1.0\n","malignant FN: 1.0\n","\n","\n","\n","Epoch: 4  val \n","Loss: 30.1320  Acc: 93.7500\n","benign precision: 100.0000  recall: 85.7143\n","benign sensitivity: 85.7143  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 90.0000\n","benign TP: 6.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 1.0\n","malignant precision: 90.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 85.7143\n","malignant FPR: 14.2857  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 6.0\n","malignant FP: 1.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 5/50\n","----------\n","Epoch: 5     train index of 5 minibatch: 1      time used: 3.6437573432922363\n","minibatch AVG loss: 31.824637985229494\n","Epoch: 5     train index of 5 minibatch: 2      time used: 3.165731430053711\n","minibatch AVG loss: 25.391119384765624\n","Epoch: 5     train index of 5 minibatch: 3      time used: 3.164531707763672\n","minibatch AVG loss: 22.95510025024414\n","\n","Epoch: 5  train \n","Loss: 26.5338  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 5  val \n","Loss: 38.5262  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 6/50\n","----------\n","Epoch: 6     train index of 5 minibatch: 1      time used: 3.646047353744507\n","minibatch AVG loss: 22.010582733154298\n","Epoch: 6     train index of 5 minibatch: 2      time used: 3.165858030319214\n","minibatch AVG loss: 24.558456230163575\n","Epoch: 6     train index of 5 minibatch: 3      time used: 3.1632113456726074\n","minibatch AVG loss: 21.381931495666503\n","\n","Epoch: 6  train \n","Loss: 22.8648  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 6  val \n","Loss: 35.4258  Acc: 93.7500\n","benign precision: 87.5000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 88.8889\n","benign FPR: 11.1111  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 8.0\n","benign FP: 1.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 88.8889\n","malignant sensitivity: 88.8889  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 87.5000\n","malignant TP: 8.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 1.0\n","\n","\n","\n","Epoch 7/50\n","----------\n","Epoch: 7     train index of 5 minibatch: 1      time used: 3.65358567237854\n","minibatch AVG loss: 31.6327917098999\n","Epoch: 7     train index of 5 minibatch: 2      time used: 3.1653809547424316\n","minibatch AVG loss: 20.34484100341797\n","Epoch: 7     train index of 5 minibatch: 3      time used: 3.166351795196533\n","minibatch AVG loss: 24.450584411621094\n","\n","Epoch: 7  train \n","Loss: 24.8019  Acc: 95.6522\n","benign precision: 93.5484  recall: 96.6667\n","benign sensitivity: 96.6667  specificity: 94.8718\n","benign FPR: 5.1282  NPV: 97.3684\n","benign TP: 29.0\n","benign TN: 37.0\n","benign FP: 2.0\n","benign FN: 1.0\n","malignant precision: 97.3684  recall: 94.8718\n","malignant sensitivity: 94.8718  specificity: 96.6667\n","malignant FPR: 3.3333  NPV: 93.5484\n","malignant TP: 37.0\n","malignant TN: 29.0\n","malignant FP: 1.0\n","malignant FN: 2.0\n","\n","\n","\n","Epoch: 7  val \n","Loss: 27.1634  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 8/50\n","----------\n","Epoch: 8     train index of 5 minibatch: 1      time used: 3.6014280319213867\n","minibatch AVG loss: 18.477080154418946\n","Epoch: 8     train index of 5 minibatch: 2      time used: 3.167576313018799\n","minibatch AVG loss: 21.159688186645507\n","Epoch: 8     train index of 5 minibatch: 3      time used: 3.1651506423950195\n","minibatch AVG loss: 31.57441864013672\n","\n","Epoch: 8  train \n","Loss: 23.2825  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 8  val \n","Loss: 31.4269  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 9/50\n","----------\n","Epoch: 9     train index of 5 minibatch: 1      time used: 3.6738977432250977\n","minibatch AVG loss: 28.231468963623048\n","Epoch: 9     train index of 5 minibatch: 2      time used: 3.1656389236450195\n","minibatch AVG loss: 15.681504440307616\n","Epoch: 9     train index of 5 minibatch: 3      time used: 3.1652050018310547\n","minibatch AVG loss: 17.516927909851074\n","\n","Epoch: 9  train \n","Loss: 20.5695  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 9  val \n","Loss: 31.7888  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 10/50\n","----------\n","Epoch: 10     train index of 5 minibatch: 1      time used: 3.6541988849639893\n","minibatch AVG loss: 13.830303573608399\n","Epoch: 10     train index of 5 minibatch: 2      time used: 3.170267105102539\n","minibatch AVG loss: 21.963394546508788\n","Epoch: 10     train index of 5 minibatch: 3      time used: 3.1665596961975098\n","minibatch AVG loss: 18.050014114379884\n","\n","Epoch: 10  train \n","Loss: 17.1651  Acc: 98.5507\n","benign precision: 96.7742  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 97.4359\n","benign FPR: 2.5641  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 38.0\n","benign FP: 1.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 97.4359\n","malignant sensitivity: 97.4359  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 96.7742\n","malignant TP: 38.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 1.0\n","\n","\n","\n","Epoch: 10  val \n","Loss: 22.2193  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 11/50\n","----------\n","Epoch: 11     train index of 5 minibatch: 1      time used: 3.624589443206787\n","minibatch AVG loss: 15.416498184204102\n","Epoch: 11     train index of 5 minibatch: 2      time used: 3.1638405323028564\n","minibatch AVG loss: 17.664760780334472\n","Epoch: 11     train index of 5 minibatch: 3      time used: 3.168666362762451\n","minibatch AVG loss: 13.913650131225586\n","\n","Epoch: 11  train \n","Loss: 15.4128  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 11  val \n","Loss: 39.2429  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 12/50\n","----------\n","Epoch: 12     train index of 5 minibatch: 1      time used: 3.6552157402038574\n","minibatch AVG loss: 28.881803894042967\n","Epoch: 12     train index of 5 minibatch: 2      time used: 3.1637320518493652\n","minibatch AVG loss: 17.562220573425293\n","Epoch: 12     train index of 5 minibatch: 3      time used: 3.1645619869232178\n","minibatch AVG loss: 16.763526153564452\n","\n","Epoch: 12  train \n","Loss: 20.0680  Acc: 98.5507\n","benign precision: 100.0000  recall: 96.6667\n","benign sensitivity: 96.6667  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 97.5000\n","benign TP: 29.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 1.0\n","malignant precision: 97.5000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 96.6667\n","malignant FPR: 3.3333  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 29.0\n","malignant FP: 1.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 12  val \n","Loss: 26.1944  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 13/50\n","----------\n","Epoch: 13     train index of 5 minibatch: 1      time used: 3.6680610179901123\n","minibatch AVG loss: 11.454651641845704\n","Epoch: 13     train index of 5 minibatch: 2      time used: 3.1630947589874268\n","minibatch AVG loss: 19.278309059143066\n","Epoch: 13     train index of 5 minibatch: 3      time used: 3.1621103286743164\n","minibatch AVG loss: 24.20158462524414\n","\n","Epoch: 13  train \n","Loss: 17.6072  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 13  val \n","Loss: 24.0545  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 14/50\n","----------\n","Epoch: 14     train index of 5 minibatch: 1      time used: 3.63401460647583\n","minibatch AVG loss: 15.460171699523926\n","Epoch: 14     train index of 5 minibatch: 2      time used: 3.1656248569488525\n","minibatch AVG loss: 15.049464416503906\n","Epoch: 14     train index of 5 minibatch: 3      time used: 3.1668848991394043\n","minibatch AVG loss: 15.044139289855957\n","\n","Epoch: 14  train \n","Loss: 15.1632  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 14  val \n","Loss: 31.8527  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 15/50\n","----------\n","Epoch: 15     train index of 5 minibatch: 1      time used: 3.639202833175659\n","minibatch AVG loss: 13.883374786376953\n","Epoch: 15     train index of 5 minibatch: 2      time used: 3.162330389022827\n","minibatch AVG loss: 14.828215217590332\n","Epoch: 15     train index of 5 minibatch: 3      time used: 3.162126302719116\n","minibatch AVG loss: 16.22034740447998\n","\n","Epoch: 15  train \n","Loss: 15.8946  Acc: 98.5507\n","benign precision: 100.0000  recall: 96.6667\n","benign sensitivity: 96.6667  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 97.5000\n","benign TP: 29.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 1.0\n","malignant precision: 97.5000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 96.6667\n","malignant FPR: 3.3333  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 29.0\n","malignant FP: 1.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 15  val \n","Loss: 29.8203  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 16/50\n","----------\n","Epoch: 16     train index of 5 minibatch: 1      time used: 3.6478960514068604\n","minibatch AVG loss: 17.223883438110352\n","Epoch: 16     train index of 5 minibatch: 2      time used: 3.1657915115356445\n","minibatch AVG loss: 21.179190826416015\n","Epoch: 16     train index of 5 minibatch: 3      time used: 3.1626551151275635\n","minibatch AVG loss: 12.932680320739745\n","\n","Epoch: 16  train \n","Loss: 16.5447  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 16  val \n","Loss: 28.3713  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 17/50\n","----------\n","Epoch: 17     train index of 5 minibatch: 1      time used: 3.6654088497161865\n","minibatch AVG loss: 15.28385009765625\n","Epoch: 17     train index of 5 minibatch: 2      time used: 3.1682615280151367\n","minibatch AVG loss: 14.089370727539062\n","Epoch: 17     train index of 5 minibatch: 3      time used: 3.1654207706451416\n","minibatch AVG loss: 14.669724655151366\n","\n","Epoch: 17  train \n","Loss: 14.6994  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 17  val \n","Loss: 24.9076  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 18/50\n","----------\n","Epoch: 18     train index of 5 minibatch: 1      time used: 3.6639041900634766\n","minibatch AVG loss: 15.391369819641113\n","Epoch: 18     train index of 5 minibatch: 2      time used: 3.1678590774536133\n","minibatch AVG loss: 17.168589210510255\n","Epoch: 18     train index of 5 minibatch: 3      time used: 3.163388967514038\n","minibatch AVG loss: 12.203887462615967\n","\n","Epoch: 18  train \n","Loss: 15.7161  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 18  val \n","Loss: 20.3762  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 19/50\n","----------\n","Epoch: 19     train index of 5 minibatch: 1      time used: 3.634384870529175\n","minibatch AVG loss: 12.590734195709228\n","Epoch: 19     train index of 5 minibatch: 2      time used: 3.1658339500427246\n","minibatch AVG loss: 13.657126998901367\n","Epoch: 19     train index of 5 minibatch: 3      time used: 3.164119005203247\n","minibatch AVG loss: 10.502670288085938\n","\n","Epoch: 19  train \n","Loss: 12.1415  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 19  val \n","Loss: 29.7363  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 20/50\n","----------\n","Epoch: 20     train index of 5 minibatch: 1      time used: 3.6545870304107666\n","minibatch AVG loss: 12.649485969543457\n","Epoch: 20     train index of 5 minibatch: 2      time used: 3.165052890777588\n","minibatch AVG loss: 13.52025489807129\n","Epoch: 20     train index of 5 minibatch: 3      time used: 3.163290500640869\n","minibatch AVG loss: 13.403569793701172\n","\n","Epoch: 20  train \n","Loss: 13.9302  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 20  val \n","Loss: 31.7120  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 21/50\n","----------\n","Epoch: 21     train index of 5 minibatch: 1      time used: 3.668043851852417\n","minibatch AVG loss: 10.153138065338135\n","Epoch: 21     train index of 5 minibatch: 2      time used: 3.1651041507720947\n","minibatch AVG loss: 13.52764434814453\n","Epoch: 21     train index of 5 minibatch: 3      time used: 3.1623730659484863\n","minibatch AVG loss: 23.15730724334717\n","\n","Epoch: 21  train \n","Loss: 17.3303  Acc: 98.5507\n","benign precision: 96.7742  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 97.4359\n","benign FPR: 2.5641  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 38.0\n","benign FP: 1.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 97.4359\n","malignant sensitivity: 97.4359  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 96.7742\n","malignant TP: 38.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 1.0\n","\n","\n","\n","Epoch: 21  val \n","Loss: 16.9633  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 22/50\n","----------\n","Epoch: 22     train index of 5 minibatch: 1      time used: 3.6749470233917236\n","minibatch AVG loss: 13.926661109924316\n","Epoch: 22     train index of 5 minibatch: 2      time used: 3.1666972637176514\n","minibatch AVG loss: 17.33890552520752\n","Epoch: 22     train index of 5 minibatch: 3      time used: 3.163752555847168\n","minibatch AVG loss: 15.857477951049805\n","\n","Epoch: 22  train \n","Loss: 15.4526  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 22  val \n","Loss: 18.5548  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 23/50\n","----------\n","Epoch: 23     train index of 5 minibatch: 1      time used: 3.641145706176758\n","minibatch AVG loss: 16.78268642425537\n","Epoch: 23     train index of 5 minibatch: 2      time used: 3.1646945476531982\n","minibatch AVG loss: 16.570593547821044\n","Epoch: 23     train index of 5 minibatch: 3      time used: 3.1646645069122314\n","minibatch AVG loss: 9.30807147026062\n","\n","Epoch: 23  train \n","Loss: 14.5644  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 23  val \n","Loss: 30.2782  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 24/50\n","----------\n","Epoch: 24     train index of 5 minibatch: 1      time used: 3.6613128185272217\n","minibatch AVG loss: 14.113422393798828\n","Epoch: 24     train index of 5 minibatch: 2      time used: 3.168076992034912\n","minibatch AVG loss: 16.064736938476564\n","Epoch: 24     train index of 5 minibatch: 3      time used: 3.1663196086883545\n","minibatch AVG loss: 11.65426139831543\n","\n","Epoch: 24  train \n","Loss: 14.2253  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 24  val \n","Loss: 32.7997  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 25/50\n","----------\n","Epoch: 25     train index of 5 minibatch: 1      time used: 3.629878044128418\n","minibatch AVG loss: 12.594373512268067\n","Epoch: 25     train index of 5 minibatch: 2      time used: 3.1653690338134766\n","minibatch AVG loss: 12.980963039398194\n","Epoch: 25     train index of 5 minibatch: 3      time used: 3.1668174266815186\n","minibatch AVG loss: 14.784428596496582\n","\n","Epoch: 25  train \n","Loss: 13.4619  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 25  val \n","Loss: 21.6732  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 26/50\n","----------\n","Epoch: 26     train index of 5 minibatch: 1      time used: 3.6522860527038574\n","minibatch AVG loss: 11.9283016204834\n","Epoch: 26     train index of 5 minibatch: 2      time used: 3.16521954536438\n","minibatch AVG loss: 12.051972961425781\n","Epoch: 26     train index of 5 minibatch: 3      time used: 3.1646511554718018\n","minibatch AVG loss: 10.93137149810791\n","\n","Epoch: 26  train \n","Loss: 12.0070  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 26  val \n","Loss: 28.9932  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 27/50\n","----------\n","Epoch: 27     train index of 5 minibatch: 1      time used: 3.682161569595337\n","minibatch AVG loss: 12.931074905395509\n","Epoch: 27     train index of 5 minibatch: 2      time used: 3.163963794708252\n","minibatch AVG loss: 10.830554962158203\n","Epoch: 27     train index of 5 minibatch: 3      time used: 3.1663198471069336\n","minibatch AVG loss: 10.964994335174561\n","\n","Epoch: 27  train \n","Loss: 11.6088  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 27  val \n","Loss: 26.3117  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 28/50\n","----------\n","Epoch: 28     train index of 5 minibatch: 1      time used: 3.6546478271484375\n","minibatch AVG loss: 12.522712898254394\n","Epoch: 28     train index of 5 minibatch: 2      time used: 3.1647443771362305\n","minibatch AVG loss: 11.768148136138915\n","Epoch: 28     train index of 5 minibatch: 3      time used: 3.164486885070801\n","minibatch AVG loss: 13.321372795104981\n","\n","Epoch: 28  train \n","Loss: 12.6596  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 28  val \n","Loss: 17.8986  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 29/50\n","----------\n","Epoch: 29     train index of 5 minibatch: 1      time used: 3.6483747959136963\n","minibatch AVG loss: 11.255674743652344\n","Epoch: 29     train index of 5 minibatch: 2      time used: 3.1637141704559326\n","minibatch AVG loss: 17.17874050140381\n","Epoch: 29     train index of 5 minibatch: 3      time used: 3.163217782974243\n","minibatch AVG loss: 11.237236595153808\n","\n","Epoch: 29  train \n","Loss: 13.7942  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 29  val \n","Loss: 20.2007  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 30/50\n","----------\n","Epoch: 30     train index of 5 minibatch: 1      time used: 3.645714282989502\n","minibatch AVG loss: 13.095143032073974\n","Epoch: 30     train index of 5 minibatch: 2      time used: 3.168464422225952\n","minibatch AVG loss: 14.316971778869629\n","Epoch: 30     train index of 5 minibatch: 3      time used: 3.1647090911865234\n","minibatch AVG loss: 13.773610496520996\n","\n","Epoch: 30  train \n","Loss: 13.2412  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 30  val \n","Loss: 26.9633  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 31/50\n","----------\n","Epoch: 31     train index of 5 minibatch: 1      time used: 3.653862714767456\n","minibatch AVG loss: 14.35668659210205\n","Epoch: 31     train index of 5 minibatch: 2      time used: 3.1654162406921387\n","minibatch AVG loss: 6.778967952728271\n","Epoch: 31     train index of 5 minibatch: 3      time used: 3.1652514934539795\n","minibatch AVG loss: 11.333904838562011\n","\n","Epoch: 31  train \n","Loss: 11.0477  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 31  val \n","Loss: 24.5420  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 32/50\n","----------\n","Epoch: 32     train index of 5 minibatch: 1      time used: 3.6403462886810303\n","minibatch AVG loss: 9.524894714355469\n","Epoch: 32     train index of 5 minibatch: 2      time used: 3.1670920848846436\n","minibatch AVG loss: 10.876092720031739\n","Epoch: 32     train index of 5 minibatch: 3      time used: 3.1660759449005127\n","minibatch AVG loss: 8.2827317237854\n","\n","Epoch: 32  train \n","Loss: 9.5584  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 32  val \n","Loss: 25.3065  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 33/50\n","----------\n","Epoch: 33     train index of 5 minibatch: 1      time used: 3.6049797534942627\n","minibatch AVG loss: 12.019367599487305\n","Epoch: 33     train index of 5 minibatch: 2      time used: 3.1634907722473145\n","minibatch AVG loss: 9.507521438598634\n","Epoch: 33     train index of 5 minibatch: 3      time used: 3.163700819015503\n","minibatch AVG loss: 10.33903980255127\n","\n","Epoch: 33  train \n","Loss: 11.0884  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 33  val \n","Loss: 24.4577  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 34/50\n","----------\n","Epoch: 34     train index of 5 minibatch: 1      time used: 3.640723705291748\n","minibatch AVG loss: 9.57025375366211\n","Epoch: 34     train index of 5 minibatch: 2      time used: 3.168874740600586\n","minibatch AVG loss: 10.522507476806641\n","Epoch: 34     train index of 5 minibatch: 3      time used: 3.166837453842163\n","minibatch AVG loss: 11.008956336975098\n","\n","Epoch: 34  train \n","Loss: 10.4154  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 34  val \n","Loss: 23.8422  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 35/50\n","----------\n","Epoch: 35     train index of 5 minibatch: 1      time used: 3.6464931964874268\n","minibatch AVG loss: 10.710340023040771\n","Epoch: 35     train index of 5 minibatch: 2      time used: 3.1663217544555664\n","minibatch AVG loss: 13.095301246643066\n","Epoch: 35     train index of 5 minibatch: 3      time used: 3.1652491092681885\n","minibatch AVG loss: 8.943057823181153\n","\n","Epoch: 35  train \n","Loss: 11.3108  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 35  val \n","Loss: 21.7632  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 36/50\n","----------\n","Epoch: 36     train index of 5 minibatch: 1      time used: 3.638617515563965\n","minibatch AVG loss: 9.91234884262085\n","Epoch: 36     train index of 5 minibatch: 2      time used: 3.1683082580566406\n","minibatch AVG loss: 12.759875869750976\n","Epoch: 36     train index of 5 minibatch: 3      time used: 3.1653521060943604\n","minibatch AVG loss: 8.671969890594482\n","\n","Epoch: 36  train \n","Loss: 10.4321  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 36  val \n","Loss: 28.0100  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 37/50\n","----------\n","Epoch: 37     train index of 5 minibatch: 1      time used: 3.612642288208008\n","minibatch AVG loss: 13.05638256072998\n","Epoch: 37     train index of 5 minibatch: 2      time used: 3.1692087650299072\n","minibatch AVG loss: 7.9477263450622555\n","Epoch: 37     train index of 5 minibatch: 3      time used: 3.1626195907592773\n","minibatch AVG loss: 11.066472434997559\n","\n","Epoch: 37  train \n","Loss: 10.4725  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 37  val \n","Loss: 25.0312  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 38/50\n","----------\n","Epoch: 38     train index of 5 minibatch: 1      time used: 3.647178888320923\n","minibatch AVG loss: 8.39669065475464\n","Epoch: 38     train index of 5 minibatch: 2      time used: 3.1655020713806152\n","minibatch AVG loss: 11.188537979125977\n","Epoch: 38     train index of 5 minibatch: 3      time used: 3.1639299392700195\n","minibatch AVG loss: 10.84281759262085\n","\n","Epoch: 38  train \n","Loss: 10.2684  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 38  val \n","Loss: 24.8483  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 39/50\n","----------\n","Epoch: 39     train index of 5 minibatch: 1      time used: 3.617187261581421\n","minibatch AVG loss: 8.367268466949463\n","Epoch: 39     train index of 5 minibatch: 2      time used: 3.164886236190796\n","minibatch AVG loss: 9.315939712524415\n","Epoch: 39     train index of 5 minibatch: 3      time used: 3.1674225330352783\n","minibatch AVG loss: 9.557971858978272\n","\n","Epoch: 39  train \n","Loss: 9.1782  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 39  val \n","Loss: 20.8542  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 40/50\n","----------\n","Epoch: 40     train index of 5 minibatch: 1      time used: 3.6254444122314453\n","minibatch AVG loss: 7.060813236236572\n","Epoch: 40     train index of 5 minibatch: 2      time used: 3.168499231338501\n","minibatch AVG loss: 8.98172492980957\n","Epoch: 40     train index of 5 minibatch: 3      time used: 3.1663317680358887\n","minibatch AVG loss: 7.803049564361572\n","\n","Epoch: 40  train \n","Loss: 8.0090  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 40  val \n","Loss: 24.0352  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 41/50\n","----------\n","Epoch: 41     train index of 5 minibatch: 1      time used: 3.6439075469970703\n","minibatch AVG loss: 8.432618522644043\n","Epoch: 41     train index of 5 minibatch: 2      time used: 3.1637258529663086\n","minibatch AVG loss: 6.727303838729858\n","Epoch: 41     train index of 5 minibatch: 3      time used: 3.1640372276306152\n","minibatch AVG loss: 9.987234210968017\n","\n","Epoch: 41  train \n","Loss: 8.6169  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 41  val \n","Loss: 21.9018  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 42/50\n","----------\n","Epoch: 42     train index of 5 minibatch: 1      time used: 3.633082866668701\n","minibatch AVG loss: 10.464530563354492\n","Epoch: 42     train index of 5 minibatch: 2      time used: 3.1645190715789795\n","minibatch AVG loss: 9.33576192855835\n","Epoch: 42     train index of 5 minibatch: 3      time used: 3.1648600101470947\n","minibatch AVG loss: 9.407290172576904\n","\n","Epoch: 42  train \n","Loss: 9.5207  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 42  val \n","Loss: 23.4256  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 43/50\n","----------\n","Epoch: 43     train index of 5 minibatch: 1      time used: 3.6304447650909424\n","minibatch AVG loss: 6.603910064697265\n","Epoch: 43     train index of 5 minibatch: 2      time used: 3.164935827255249\n","minibatch AVG loss: 12.468148612976075\n","Epoch: 43     train index of 5 minibatch: 3      time used: 3.1654727458953857\n","minibatch AVG loss: 7.5396932601928714\n","\n","Epoch: 43  train \n","Loss: 8.6600  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 43  val \n","Loss: 23.7400  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 44/50\n","----------\n","Epoch: 44     train index of 5 minibatch: 1      time used: 3.6469433307647705\n","minibatch AVG loss: 9.121467399597169\n","Epoch: 44     train index of 5 minibatch: 2      time used: 3.168567180633545\n","minibatch AVG loss: 8.44074878692627\n","Epoch: 44     train index of 5 minibatch: 3      time used: 3.171989917755127\n","minibatch AVG loss: 8.909481239318847\n","\n","Epoch: 44  train \n","Loss: 8.8281  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 44  val \n","Loss: 24.7561  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 45/50\n","----------\n","Epoch: 45     train index of 5 minibatch: 1      time used: 3.6444966793060303\n","minibatch AVG loss: 6.951697540283203\n","Epoch: 45     train index of 5 minibatch: 2      time used: 3.164384126663208\n","minibatch AVG loss: 11.662968254089355\n","Epoch: 45     train index of 5 minibatch: 3      time used: 3.1669890880584717\n","minibatch AVG loss: 9.209070491790772\n","\n","Epoch: 45  train \n","Loss: 9.2609  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 45  val \n","Loss: 24.2358  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 46/50\n","----------\n","Epoch: 46     train index of 5 minibatch: 1      time used: 3.644568920135498\n","minibatch AVG loss: 8.685888862609863\n","Epoch: 46     train index of 5 minibatch: 2      time used: 3.1706490516662598\n","minibatch AVG loss: 10.1112850189209\n","Epoch: 46     train index of 5 minibatch: 3      time used: 3.168160915374756\n","minibatch AVG loss: 8.08840503692627\n","\n","Epoch: 46  train \n","Loss: 8.5847  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 46  val \n","Loss: 21.6887  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 47/50\n","----------\n","Epoch: 47     train index of 5 minibatch: 1      time used: 3.6232492923736572\n","minibatch AVG loss: 8.235798454284668\n","Epoch: 47     train index of 5 minibatch: 2      time used: 3.165498733520508\n","minibatch AVG loss: 6.037417888641357\n","Epoch: 47     train index of 5 minibatch: 3      time used: 3.1669538021087646\n","minibatch AVG loss: 10.474438381195068\n","\n","Epoch: 47  train \n","Loss: 8.0190  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 47  val \n","Loss: 24.3624  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 48/50\n","----------\n","Epoch: 48     train index of 5 minibatch: 1      time used: 3.645012378692627\n","minibatch AVG loss: 6.929254150390625\n","Epoch: 48     train index of 5 minibatch: 2      time used: 3.1673829555511475\n","minibatch AVG loss: 7.617064094543457\n","Epoch: 48     train index of 5 minibatch: 3      time used: 3.1665940284729004\n","minibatch AVG loss: 13.03050594329834\n","\n","Epoch: 48  train \n","Loss: 8.8498  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 48  val \n","Loss: 21.5719  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 49/50\n","----------\n","Epoch: 49     train index of 5 minibatch: 1      time used: 3.6299235820770264\n","minibatch AVG loss: 7.375674915313721\n","Epoch: 49     train index of 5 minibatch: 2      time used: 3.1641008853912354\n","minibatch AVG loss: 10.456915855407715\n","Epoch: 49     train index of 5 minibatch: 3      time used: 3.1661906242370605\n","minibatch AVG loss: 7.360659217834472\n","\n","Epoch: 49  train \n","Loss: 8.1086  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 49  val \n","Loss: 22.7013  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch 50/50\n","----------\n","Epoch: 50     train index of 5 minibatch: 1      time used: 3.6370205879211426\n","minibatch AVG loss: 7.6142960548400875\n","Epoch: 50     train index of 5 minibatch: 2      time used: 3.165433406829834\n","minibatch AVG loss: 10.119829845428466\n","Epoch: 50     train index of 5 minibatch: 3      time used: 3.1661324501037598\n","minibatch AVG loss: 6.954656982421875\n","\n","Epoch: 50  train \n","Loss: 8.6133  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 30.0\n","benign TN: 39.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 39.0\n","malignant TN: 30.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Epoch: 50  val \n","Loss: 23.6926  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 7.0\n","benign TN: 9.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 9.0\n","malignant TN: 7.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","\n","Training complete in 10m 36s\n","Best epoch idx:  50\n","Best epoch train Acc: 100.000000\n","Best epoch val Acc: 100.000000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","model trained by GPU (idx:0) has been saved at  /home/MIL_Experiment/saved_models/MIL_ViT_384_401_PT_lf05_b4_p32_warwick_MIL.pth\n"]}],"source":["!python MIL_train.py --model_idx ViT_384_401_PT_lf05_b4_p32_warwick_MIL --data_augmentation_mode 2 --edge_size 384 --batch_size 4 --patch_size 32 --lr 0.00001 --lrf 0.05 --enable_tensorboard --check_minibatch 5 --dataroot /data/MIL_Experiment/dataset/warwick_MIL --model_path /home/MIL_Experiment/saved_models --draw_root /home/MIL_Experiment/runs"]},{"cell_type":"markdown","metadata":{"id":"DLy5OE1kOq3W"},"source":["* MIL Test"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"IxmTlFRIV4lu"},"outputs":[{"name":"stdout","output_type":"stream","text":["*********************************setting*************************************\n","Namespace(CLS_MIL_head_weight=1.0, CLS_MIL_off=False, MIL_Stripe=False, MIL_head_weight=1.0, Pre_Trained_model_path=None, batch_size=1, check_minibatch=5, data_augmentation_mode=2, dataroot='/data/MIL_Experiment/dataset/warwick_MIL', draw_root='/home/MIL_Experiment/runs', edge_size=384, enable_attention_check=False, enable_notify=False, enable_tensorboard=False, enable_visualize_check=False, gpu_idx=0, model_idx='ViT_384_401_PT_lf05_b4_p32_warwick_MIL', model_path='/home/MIL_Experiment/saved_models', paint=True, patch_size=32, shuffle_MIL_off=False, shuffle_attention_check=False, shuffle_dataloader=False)\n","GPU: 0\n","model loaded\n","model : ViT_384_401_PT_lf05_b4_p32_warwick_MIL\n","Epoch: Test\n","----------\n","Epoch: test     test index of 5 minibatch: 1      time used: 0.4074726104736328\n","minibatch AVG loss: 25.286715698242187\n","Epoch: test     test index of 5 minibatch: 2      time used: 0.2811000347137451\n","minibatch AVG loss: 14.636392593383789\n","Epoch: test     test index of 5 minibatch: 3      time used: 0.2799532413482666\n","minibatch AVG loss: 9.237878894805908\n","Epoch: test     test index of 5 minibatch: 4      time used: 0.2799546718597412\n","minibatch AVG loss: 16.125740432739256\n","Epoch: test     test index of 5 minibatch: 5      time used: 0.27948570251464844\n","minibatch AVG loss: 27.6292884349823\n","Epoch: test     test index of 5 minibatch: 6      time used: 0.2818284034729004\n","minibatch AVG loss: 12.4674503326416\n","Epoch: test     test index of 5 minibatch: 7      time used: 0.27991414070129395\n","minibatch AVG loss: 24.90966691970825\n","Epoch: test     test index of 5 minibatch: 8      time used: 0.2798495292663574\n","minibatch AVG loss: 25.039684104919434\n","Epoch: test     test index of 5 minibatch: 9      time used: 0.28023767471313477\n","minibatch AVG loss: 16.2663330078125\n","Epoch: test     test index of 5 minibatch: 10      time used: 0.28078532218933105\n","minibatch AVG loss: 31.84072265625\n","Epoch: test     test index of 5 minibatch: 11      time used: 0.2795698642730713\n","minibatch AVG loss: 44.595933532714845\n","Epoch: test     test index of 5 minibatch: 12      time used: 0.2796967029571533\n","minibatch AVG loss: 25.991370964050294\n","Epoch: test     test index of 5 minibatch: 13      time used: 0.28062963485717773\n","minibatch AVG loss: 11.937229251861572\n","Epoch: test     test index of 5 minibatch: 14      time used: 0.2799949645996094\n","minibatch AVG loss: 22.776964282989503\n","Epoch: test     test index of 5 minibatch: 15      time used: 0.2806377410888672\n","minibatch AVG loss: 25.77069206237793\n","Epoch: test     test index of 5 minibatch: 16      time used: 0.28052830696105957\n","minibatch AVG loss: 23.203466796875\n","\n","Epoch:  test \n","Loss: 22.3572  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 37.0\n","benign TN: 43.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 43.0\n","malignant TN: 37.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","Testing complete in 0m 5s\n"]}],"source":["!python MIL_test.py --model_idx ViT_384_401_PT_lf05_b4_p32_warwick_MIL --data_augmentation_mode 2 --edge_size 384 --batch_size 1 --patch_size 32  --check_minibatch 5 --dataroot /data/MIL_Experiment/dataset/warwick_MIL --model_path /home/MIL_Experiment/saved_models --draw_root /home/MIL_Experiment/runs"]},{"cell_type":"markdown","metadata":{"id":"2dypseRcOtHU"},"source":["# Test \u0026 Imaging"]},{"cell_type":"markdown","metadata":{"id":"HS1jyFoHSeih"},"source":["Basic CLS CAM"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"v5WbvtSpZXyC"},"outputs":[{"name":"stdout","output_type":"stream","text":["class_names: ['benign', 'malignant']\n","model loaded\n","model : ViT_384_401_PT_lf05_b4_p32_warwick_MIL\n","*********************************setting*************************************\n","Namespace(MIL_Stripe=True, att_module='SimAM', attn_drop_rate=0.0, batch_size=1, check_minibatch=5, cls_token_off=False, data_augmentation_mode=2, dataroot='/data/MIL_Experiment/dataset/warwick_CLS', draw_root='/home/MIL_Experiment/imaging_results', drop_path_rate=0.0, drop_rate=0.0, edge_size=384, enable_attention_check=True, enable_notify=False, enable_tensorboard=False, enable_visualize_check=False, gpu_idx=0, model_idx='ViT_384_401_PT_lf05_b4_p32_warwick_MIL', model_path='/home/MIL_Experiment/saved_models', num_classes=0, paint=True, pos_embedding_off=False)\n","Epoch: Test\n","----------\n","Epoch: test     test index of 5 minibatch: 1      time used: 0.2544424533843994\n","minibatch AVG loss: 0.002735888137249276\n","/home/MIL_Experiment/code/utils/visual_usage.py:18: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n","  return sof(x)\n","Epoch: test     test index of 5 minibatch: 2      time used: 0.1416640281677246\n","minibatch AVG loss: 8.959356491686777e-05\n","Epoch: test     test index of 5 minibatch: 3      time used: 0.13994860649108887\n","minibatch AVG loss: 0.00016623363335384057\n","Epoch: test     test index of 5 minibatch: 4      time used: 0.13986468315124512\n","minibatch AVG loss: 0.0004909527575364337\n","Epoch: test     test index of 5 minibatch: 5      time used: 0.1386866569519043\n","minibatch AVG loss: 0.0007842307742976118\n","Epoch: test     test index of 5 minibatch: 6      time used: 0.14126873016357422\n","minibatch AVG loss: 7.824521344446111e-05\n","Epoch: test     test index of 5 minibatch: 7      time used: 0.1422410011291504\n","minibatch AVG loss: 0.0026375615867436863\n","Epoch: test     test index of 5 minibatch: 8      time used: 0.13952946662902832\n","minibatch AVG loss: 0.00042418737430125473\n","Epoch: test     test index of 5 minibatch: 9      time used: 0.13957858085632324\n","minibatch AVG loss: 0.0007291833258932457\n","Epoch: test     test index of 5 minibatch: 10      time used: 0.13964271545410156\n","minibatch AVG loss: 0.000672500123619102\n","Epoch: test     test index of 5 minibatch: 11      time used: 0.14188694953918457\n","minibatch AVG loss: 0.0015465400851098821\n","Epoch: test     test index of 5 minibatch: 12      time used: 0.14057040214538574\n","minibatch AVG loss: 0.0006952290597837419\n","Epoch: test     test index of 5 minibatch: 13      time used: 0.1403493881225586\n","minibatch AVG loss: 0.00024265948959509843\n","Epoch: test     test index of 5 minibatch: 14      time used: 0.1394059658050537\n","minibatch AVG loss: 0.001799797597050201\n","Epoch: test     test index of 5 minibatch: 15      time used: 0.13909268379211426\n","minibatch AVG loss: 0.00016865608049556612\n","Epoch: test     test index of 5 minibatch: 16      time used: 0.13994359970092773\n","minibatch AVG loss: 0.00015633660259481986\n","\n","Epoch:  test \n","Loss: 0.0008  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 37.0\n","benign TN: 43.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 43.0\n","malignant TN: 37.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","Testing complete in 0m 30s\n"]}],"source":["!python Test.py --model_idx ViT_384_401_PT_lf05_b4_p32_warwick_MIL --data_augmentation_mode 2 --MIL_Stripe --enable_attention_check --check_minibatch 5 --edge_size 384 --dataroot /data/MIL_Experiment/dataset/warwick_CLS --model_path /home/MIL_Experiment/saved_models --draw_root /home/MIL_Experiment/imaging_results"]},{"cell_type":"markdown","metadata":{"id":"zySxjXSgSg82"},"source":["CAM on shuffled images (random batch)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"VrEyVWowSRPR"},"outputs":[{"name":"stdout","output_type":"stream","text":["*********************************setting*************************************\n","Namespace(CLS_MIL_head_weight=1.0, CLS_MIL_off=False, MIL_Stripe=True, MIL_head_weight=1.0, Pre_Trained_model_path=None, batch_size=4, check_minibatch=5, data_augmentation_mode=2, dataroot='/data/MIL_Experiment/dataset/warwick_MIL', draw_root='/home/MIL_Experiment/imaging_results', edge_size=384, enable_attention_check=False, enable_notify=False, enable_tensorboard=False, enable_visualize_check=False, gpu_idx=0, model_idx='ViT_384_401_PT_lf05_b4_p32_warwick_MIL', model_path='/home/MIL_Experiment/saved_models', paint=True, patch_size=32, shuffle_MIL_off=False, shuffle_attention_check=True, shuffle_dataloader=True)\n","GPU: 0\n","model loaded\n","model : ViT_384_401_PT_lf05_b4_p32_warwick_MIL\n","Epoch: Test\n","----------\n","Epoch: test     test index of 5 minibatch: 1      time used: 0.7422976493835449\n","minibatch AVG loss: 0.0006218856753548608\n","/home/MIL_Experiment/code/utils/visual_usage.py:18: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n","  return sof(x)\n","Epoch: test     test index of 5 minibatch: 2      time used: 0.515373706817627\n","minibatch AVG loss: 0.00124292554683052\n","Epoch: test     test index of 5 minibatch: 3      time used: 0.518047571182251\n","minibatch AVG loss: 0.0002377453405642882\n","Epoch: test     test index of 5 minibatch: 4      time used: 0.5147418975830078\n","minibatch AVG loss: 0.0016361153277102858\n","\n","Epoch:  test \n","Loss: 0.0009  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 37.0\n","benign TN: 43.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 43.0\n","malignant TN: 37.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","Testing complete in 0m 11s\n"]}],"source":["!python MIL_test.py --model_idx ViT_384_401_PT_lf05_b4_p32_warwick_MIL --data_augmentation_mode 2 --shuffle_attention_check --MIL_Stripe --edge_size 384 --shuffle_dataloader --batch_size 4 --patch_size 32 --check_minibatch 5 --dataroot /data/MIL_Experiment/dataset/warwick_MIL --model_path /home/MIL_Experiment/saved_models --draw_root /home/MIL_Experiment/imaging_results"]},{"cell_type":"markdown","metadata":{"id":"TuGZrdkvSsql"},"source":["CAM on shuffled images (continuous batch)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"0jWUVs29S6ZR"},"outputs":[{"name":"stdout","output_type":"stream","text":["*********************************setting*************************************\n","Namespace(CLS_MIL_head_weight=1.0, CLS_MIL_off=False, MIL_Stripe=True, MIL_head_weight=1.0, Pre_Trained_model_path=None, batch_size=4, check_minibatch=5, data_augmentation_mode=2, dataroot='/data/MIL_Experiment/dataset/warwick_MIL', draw_root='/home/MIL_Experiment/imaging_results', edge_size=384, enable_attention_check=False, enable_notify=False, enable_tensorboard=False, enable_visualize_check=False, gpu_idx=0, model_idx='ViT_384_401_PT_lf05_b4_p32_warwick_MIL', model_path='/home/MIL_Experiment/saved_models', paint=True, patch_size=32, shuffle_MIL_off=False, shuffle_attention_check=True, shuffle_dataloader=False)\n","GPU: 0\n","model loaded\n","model : ViT_384_401_PT_lf05_b4_p32_warwick_MIL\n","Epoch: Test\n","----------\n","Epoch: test     test index of 5 minibatch: 1      time used: 0.7266407012939453\n","minibatch AVG loss: 0.0010575442924164236\n","/home/MIL_Experiment/code/utils/visual_usage.py:18: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n","  return sof(x)\n","Epoch: test     test index of 5 minibatch: 2      time used: 0.5161828994750977\n","minibatch AVG loss: 0.0009971470310119912\n","Epoch: test     test index of 5 minibatch: 3      time used: 0.5189378261566162\n","minibatch AVG loss: 0.0010128648136742413\n","Epoch: test     test index of 5 minibatch: 4      time used: 0.5140724182128906\n","minibatch AVG loss: 0.0006711158988764509\n","\n","Epoch:  test \n","Loss: 0.0009  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 37.0\n","benign TN: 43.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 43.0\n","malignant TN: 37.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","Testing complete in 0m 11s\n"]}],"source":["!python MIL_test.py --model_idx ViT_384_401_PT_lf05_b4_p32_warwick_MIL --data_augmentation_mode 2 --shuffle_attention_check --MIL_Stripe --edge_size 384 --batch_size 4 --patch_size 32 --check_minibatch 5 --dataroot /data/MIL_Experiment/dataset/warwick_MIL --model_path /home/MIL_Experiment/saved_models --draw_root /home/MIL_Experiment/imaging_results"]},{"cell_type":"markdown","metadata":{"id":"sCmllSS8TAnD"},"source":["CAM on shuffled images (batch size = 1)"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"9KcJHlZkTHHI"},"outputs":[{"name":"stdout","output_type":"stream","text":["*********************************setting*************************************\n","Namespace(CLS_MIL_head_weight=1.0, CLS_MIL_off=False, MIL_Stripe=True, MIL_head_weight=1.0, Pre_Trained_model_path=None, batch_size=1, check_minibatch=5, data_augmentation_mode=2, dataroot='/data/MIL_Experiment/dataset/warwick_MIL', draw_root='/home/MIL_Experiment/imaging_results', edge_size=384, enable_attention_check=False, enable_notify=False, enable_tensorboard=False, enable_visualize_check=False, gpu_idx=0, model_idx='ViT_384_401_PT_lf05_b4_p32_warwick_MIL', model_path='/home/MIL_Experiment/saved_models', paint=True, patch_size=32, shuffle_MIL_off=False, shuffle_attention_check=True, shuffle_dataloader=False)\n","GPU: 0\n","model loaded\n","model : ViT_384_401_PT_lf05_b4_p32_warwick_MIL\n","Epoch: Test\n","----------\n","Epoch: test     test index of 5 minibatch: 1      time used: 0.26824474334716797\n","minibatch AVG loss: 0.0035511413705535235\n","/home/MIL_Experiment/code/utils/visual_usage.py:18: UserWarning: Implicit dimension choice for softmax has been deprecated. Change the call to include dim=X as an argument.\n","  return sof(x)\n","Epoch: test     test index of 5 minibatch: 2      time used: 0.14859485626220703\n","minibatch AVG loss: 8.809165592538193e-05\n","Epoch: test     test index of 5 minibatch: 3      time used: 0.14740729331970215\n","minibatch AVG loss: 0.0001587724924320355\n","Epoch: test     test index of 5 minibatch: 4      time used: 0.1506059169769287\n","minibatch AVG loss: 0.00043219511135248465\n","Epoch: test     test index of 5 minibatch: 5      time used: 0.15032005310058594\n","minibatch AVG loss: 0.0006763713907275814\n","Epoch: test     test index of 5 minibatch: 6      time used: 0.15018129348754883\n","minibatch AVG loss: 7.869811852287967e-05\n","Epoch: test     test index of 5 minibatch: 7      time used: 0.14756226539611816\n","minibatch AVG loss: 0.002740003491635434\n","Epoch: test     test index of 5 minibatch: 8      time used: 0.14787817001342773\n","minibatch AVG loss: 0.0004935150362143758\n","Epoch: test     test index of 5 minibatch: 9      time used: 0.14771771430969238\n","minibatch AVG loss: 0.0007884161313995719\n","Epoch: test     test index of 5 minibatch: 10      time used: 0.1466512680053711\n","minibatch AVG loss: 0.0007549386609753128\n","Epoch: test     test index of 5 minibatch: 11      time used: 0.14667725563049316\n","minibatch AVG loss: 0.0017370080517139287\n","Epoch: test     test index of 5 minibatch: 12      time used: 0.1444535255432129\n","minibatch AVG loss: 0.0007710725243669003\n","Epoch: test     test index of 5 minibatch: 13      time used: 0.148665189743042\n","minibatch AVG loss: 0.0002697047944820952\n","Epoch: test     test index of 5 minibatch: 14      time used: 0.14818811416625977\n","minibatch AVG loss: 0.0020098514229175634\n","Epoch: test     test index of 5 minibatch: 15      time used: 0.14828181266784668\n","minibatch AVG loss: 0.00021513049432542175\n","Epoch: test     test index of 5 minibatch: 16      time used: 0.14577150344848633\n","minibatch AVG loss: 0.0001897767826449126\n","\n","Epoch:  test \n","Loss: 0.0009  Acc: 100.0000\n","benign precision: 100.0000  recall: 100.0000\n","benign sensitivity: 100.0000  specificity: 100.0000\n","benign FPR: 0.0000  NPV: 100.0000\n","benign TP: 37.0\n","benign TN: 43.0\n","benign FP: 0.0\n","benign FN: 0.0\n","malignant precision: 100.0000  recall: 100.0000\n","malignant sensitivity: 100.0000  specificity: 100.0000\n","malignant FPR: 0.0000  NPV: 100.0000\n","malignant TP: 43.0\n","malignant TN: 37.0\n","malignant FP: 0.0\n","malignant FN: 0.0\n","\n","\n","Testing complete in 0m 30s\n"]}],"source":["!python MIL_test.py --model_idx ViT_384_401_PT_lf05_b4_p32_warwick_MIL --data_augmentation_mode 2 --shuffle_attention_check --MIL_Stripe --edge_size 384 --batch_size 1 --patch_size 32 --check_minibatch 5 --dataroot /data/MIL_Experiment/dataset/warwick_MIL --model_path /home/MIL_Experiment/saved_models --draw_root /home/MIL_Experiment/imaging_results"]},{"cell_type":"markdown","metadata":{"id":"32lV43PnKVJx"},"source":["# Synchronize files"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"mCZDUffdQep4"},"outputs":[],"source":["# change working dir\n","import os\n","os.chdir(\"/home/MIL_Experiment/code/utils\")\n","!python check_log_json.py --draw_root /home/MIL_Experiment/runs --record_dir /home/MIL_Experiment/CSV_logs"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"_Wx0ymiiEuyS"},"outputs":[{"name":"stdout","output_type":"stream","text":["mkdir: cannot create directory ‘/content/drive/MyDrive/MIL_SI_sample’: File exists\n","results copy completed!\n"]}],"source":["# create path on google drive\n","!mkdir /content/drive/MyDrive/MIL_SI_sample\n","# copy the results\n","!/bin/cp -rf /home/MIL_Experiment/* /content/drive/MyDrive/MIL_SI_sample/\n","print('results copy completed!')"]},{"cell_type":"code","execution_count":null,"metadata":{"colab":{"background_save":true},"id":"9lzAtLIhnGe5"},"outputs":[{"name":"stdout","output_type":"stream","text":["Mon Aug 15 18:41:10 UTC 2022\n"]}],"source":["!date --date='+8 hour'  # CST time zone"]}],"metadata":{"accelerator":"GPU","colab":{"collapsed_sections":[],"machine_shape":"hm","name":"OpenSourse Sample MIL warwick_Experiment 384 401 lf05 p32 MIL Train.ipynb","version":""},"kernelspec":{"display_name":"Python 3","name":"python3"},"language_info":{"name":"python"}},"nbformat":4,"nbformat_minor":0}