4_test_model.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "7871d385-5a16-45d7-979e-cb9e0dc631f9",
   "metadata": {},
   "source": [
    "## Help"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "e6ab256a-7f8b-43df-aca1-ed6661785fac",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Usage: supportFiles/evaluate.py <MAX_NUM_FILES> <FEATURE_SET> <PCAP_SOURCE> [\"KEEP\"] [\"SCAN_ALL\"] [\"SCAN_ONLY\"]\n",
      "Datasets available are :\n",
      " {0: 'AB-TRAP', 1: 'NB15', 2: 'CIC-IDS', 3: 'ToN-IoT', 4: 'BoT-IoT', 5: 'internet'}\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/evaluate.py"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ad66c487-01a3-41b4-9af1-c0b20cb38fd0",
   "metadata": {},
   "source": [
    "### Test conditions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1d8f0bee-f672-4c65-a59f-2540f621895c",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "TYPE_OF_TEST = \"\\\"KEEP\\\" \\\"SCAN_ONLY\\\" \\\"undersampling\\\"\" # \\\"oversampling\\\" \\\"SCAN_ALL\\\"\n",
    "MAX_NUM_FILES = 96\n",
    "CIC_FEATURES = 1\n",
    "METHOD = \"evaluateBinary.py\" #\"evaluate.py\""
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1137014a-a095-498c-8433-a7170ad01d47",
   "metadata": {},
   "source": [
    "### RUN ALL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "d53a6325-6dd1-45cd-b32d-f58100aa8192",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "Using models with undersampled majority class\n",
      "loading models from SCAN_AB-TRAP_CIC_us\n",
      "Models fetched: ['SCAN_AB-TRAP_CIC_us_DT.joblib', 'SCAN_AB-TRAP_CIC_us_LR.joblib', 'SCAN_AB-TRAP_CIC_us_MLP.joblib', 'SCAN_AB-TRAP_CIC_us_NB.joblib', 'SCAN_AB-TRAP_CIC_us_SVM.joblib', 'SCAN_AB-TRAP_CIC_us_XGB.joblib']\n",
      "DT's index of best performance: 2\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "F1-score file for SCAN_AB-TRAP_CIC_us data set not found. Creating..\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-03-14 06:25:29\n",
      "Evaluating SCAN_AB-TRAP_CIC_us's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's TN = 391, FP = 36284, FN = 1, TP = 400\n",
      "2022-03-14 06:25:29\n",
      "F1-score for DT: TN = 391, FP = 36284, FN = 1, TP = 400\n",
      "\n",
      "LR's TN = 29573, FP = 7102, FN = 401, TP = 0\n",
      "2022-03-14 06:25:29\n",
      "F1-score for LR: TN = 29573, FP = 7102, FN = 401, TP = 0\n",
      "\n",
      "MLP's TN = 10558, FP = 26117, FN = 303, TP = 98\n",
      "2022-03-14 06:25:29\n",
      "F1-score for MLP: TN = 10558, FP = 26117, FN = 303, TP = 98\n",
      "\n",
      "NB's TN = 19139, FP = 17536, FN = 308, TP = 93\n",
      "2022-03-14 06:25:29\n",
      "F1-score for NB: TN = 19139, FP = 17536, FN = 308, TP = 93\n",
      "\n",
      "SVM's TN = 22219, FP = 14456, FN = 319, TP = 82\n",
      "2022-03-14 06:25:29\n",
      "F1-score for SVM: TN = 22219, FP = 14456, FN = 319, TP = 82\n",
      "\n",
      "XGB's TN = 2589, FP = 34086, FN = 0, TP = 401\n",
      "2022-03-14 06:25:29\n",
      "F1-score for XGB: TN = 2589, FP = 34086, FN = 0, TP = 401\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-03-14 06:25:53\n",
      "Evaluating SCAN_AB-TRAP_CIC_us's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's TN = 722593, FP = 1548529, FN = 158621, TP = 183\n",
      "2022-03-14 06:25:57\n",
      "F1-score for DT: TN = 722593, FP = 1548529, FN = 158621, TP = 183\n",
      "\n",
      "LR's TN = 980224, FP = 1290898, FN = 929, TP = 157875\n",
      "2022-03-14 06:26:00\n",
      "F1-score for LR: TN = 980224, FP = 1290898, FN = 929, TP = 157875\n",
      "\n",
      "MLP's TN = 1377696, FP = 893426, FN = 407, TP = 158397\n",
      "2022-03-14 06:26:03\n",
      "F1-score for MLP: TN = 1377696, FP = 893426, FN = 407, TP = 158397\n",
      "\n",
      "NB's TN = 683547, FP = 1587575, FN = 1356, TP = 157448\n",
      "2022-03-14 06:26:09\n",
      "F1-score for NB: TN = 683547, FP = 1587575, FN = 1356, TP = 157448\n",
      "\n",
      "SVM's TN = 1210889, FP = 1060233, FN = 438, TP = 158366\n",
      "2022-03-14 06:26:12\n",
      "F1-score for SVM: TN = 1210889, FP = 1060233, FN = 438, TP = 158366\n",
      "\n",
      "XGB's TN = 696875, FP = 1574247, FN = 80037, TP = 78767\n",
      "2022-03-14 06:26:18\n",
      "F1-score for XGB: TN = 696875, FP = 1574247, FN = 80037, TP = 78767\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-03-14 06:27:08\n",
      "Evaluating SCAN_AB-TRAP_CIC_us's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's TN = 1797328, FP = 716731, FN = 4977, TP = 31228\n",
      "2022-03-14 06:27:12\n",
      "F1-score for DT: TN = 1797328, FP = 716731, FN = 4977, TP = 31228\n",
      "\n",
      "LR's TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "2022-03-14 06:27:15\n",
      "F1-score for LR: TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "\n",
      "MLP's TN = 2142550, FP = 371509, FN = 33231, TP = 2974\n",
      "2022-03-14 06:27:19\n",
      "F1-score for MLP: TN = 2142550, FP = 371509, FN = 33231, TP = 2974\n",
      "\n",
      "NB's TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "2022-03-14 06:27:25\n",
      "F1-score for NB: TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "\n",
      "SVM's TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "2022-03-14 06:27:28\n",
      "F1-score for SVM: TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "\n",
      "XGB's TN = 538000, FP = 1976059, FN = 10905, TP = 25300\n",
      "2022-03-14 06:27:34\n",
      "F1-score for XGB: TN = 538000, FP = 1976059, FN = 10905, TP = 25300\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-03-14 06:30:23\n",
      "Evaluating SCAN_AB-TRAP_CIC_us's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's TN = 72679, FP = 16555, FN = 2379522, TP = 1133888\n",
      "2022-03-14 06:30:31\n",
      "F1-score for DT: TN = 72679, FP = 16555, FN = 2379522, TP = 1133888\n",
      "\n",
      "LR's TN = 39226, FP = 50008, FN = 1655980, TP = 1857430\n",
      "2022-03-14 06:30:37\n",
      "F1-score for LR: TN = 39226, FP = 50008, FN = 1655980, TP = 1857430\n",
      "\n",
      "MLP's TN = 42320, FP = 46914, FN = 1273252, TP = 2240158\n",
      "2022-03-14 06:30:44\n",
      "F1-score for MLP: TN = 42320, FP = 46914, FN = 1273252, TP = 2240158\n",
      "\n",
      "NB's TN = 46020, FP = 43214, FN = 2694442, TP = 818968\n",
      "2022-03-14 06:30:54\n",
      "F1-score for NB: TN = 46020, FP = 43214, FN = 2694442, TP = 818968\n",
      "\n",
      "SVM's TN = 38824, FP = 50410, FN = 1559589, TP = 1953821\n",
      "2022-03-14 06:30:58\n",
      "F1-score for SVM: TN = 38824, FP = 50410, FN = 1559589, TP = 1953821\n",
      "\n",
      "XGB's TN = 55020, FP = 34214, FN = 1934734, TP = 1578676\n",
      "2022-03-14 06:31:08\n",
      "F1-score for XGB: TN = 55020, FP = 34214, FN = 1934734, TP = 1578676\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-03-14 06:33:11\n",
      "Evaluating SCAN_AB-TRAP_CIC_us's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's TN = 9105696, FP = 26657, FN = 79228, TP = 6308\n",
      "2022-03-14 06:33:26\n",
      "F1-score for DT: TN = 9105696, FP = 26657, FN = 79228, TP = 6308\n",
      "\n",
      "LR's TN = 9062024, FP = 70329, FN = 85533, TP = 3\n",
      "2022-03-14 06:33:38\n",
      "F1-score for LR: TN = 9062024, FP = 70329, FN = 85533, TP = 3\n",
      "\n",
      "MLP's TN = 9089964, FP = 42389, FN = 82506, TP = 3030\n",
      "2022-03-14 06:33:52\n",
      "F1-score for MLP: TN = 9089964, FP = 42389, FN = 82506, TP = 3030\n",
      "\n",
      "NB's TN = 1972809, FP = 7159544, FN = 4739, TP = 80797\n",
      "2022-03-14 06:34:50\n",
      "F1-score for NB: TN = 1972809, FP = 7159544, FN = 4739, TP = 80797\n",
      "\n",
      "SVM's TN = 9087836, FP = 44517, FN = 85536, TP = 0\n",
      "2022-03-14 06:35:13\n",
      "F1-score for SVM: TN = 9087836, FP = 44517, FN = 85536, TP = 0\n",
      "\n",
      "XGB's TN = 9106782, FP = 25571, FN = 80539, TP = 4997\n",
      "2022-03-14 06:35:39\n",
      "F1-score for XGB: TN = 9106782, FP = 25571, FN = 80539, TP = 4997\n",
      "\n",
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "Using models with undersampled majority class\n",
      "loading models from SCAN_NB15_CIC_us\n",
      "Models fetched: ['SCAN_NB15_CIC_us_DT.joblib', 'SCAN_NB15_CIC_us_LR.joblib', 'SCAN_NB15_CIC_us_MLP.joblib', 'SCAN_NB15_CIC_us_NB.joblib', 'SCAN_NB15_CIC_us_SVM.joblib', 'SCAN_NB15_CIC_us_XGB.joblib']\n",
      "DT's index of best performance: 0\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "F1-score file for SCAN_NB15_CIC_us data set not found. Creating..\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-03-14 06:37:57\n",
      "Evaluating SCAN_NB15_CIC_us's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's TN = 7845521, FP = 1286832, FN = 161507, TP = 516\n",
      "2022-03-14 06:38:11\n",
      "F1-score for DT: TN = 7845521, FP = 1286832, FN = 161507, TP = 516\n",
      "\n",
      "LR's TN = 2220995, FP = 6911358, FN = 153835, TP = 8188\n",
      "2022-03-14 06:38:23\n",
      "F1-score for LR: TN = 2220995, FP = 6911358, FN = 153835, TP = 8188\n",
      "\n",
      "MLP's TN = 923570, FP = 8208783, FN = 146908, TP = 15115\n",
      "2022-03-14 06:38:37\n",
      "F1-score for MLP: TN = 923570, FP = 8208783, FN = 146908, TP = 15115\n",
      "\n",
      "NB's TN = 7983602, FP = 1148751, FN = 160477, TP = 1546\n",
      "2022-03-14 06:45:19\n",
      "F1-score for NB: TN = 7983602, FP = 1148751, FN = 160477, TP = 1546\n",
      "\n",
      "SVM's TN = 3460832, FP = 5671521, FN = 157172, TP = 4851\n",
      "2022-03-14 06:46:03\n",
      "F1-score for SVM: TN = 3460832, FP = 5671521, FN = 157172, TP = 4851\n",
      "\n",
      "XGB's TN = 7867853, FP = 1264500, FN = 161688, TP = 335\n",
      "2022-03-14 06:46:32\n",
      "F1-score for XGB: TN = 7867853, FP = 1264500, FN = 161688, TP = 335\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-03-14 06:48:51\n",
      "Evaluating SCAN_NB15_CIC_us's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's TN = 1162226, FP = 1108896, FN = 148145, TP = 10659\n",
      "2022-03-14 06:48:54\n",
      "F1-score for DT: TN = 1162226, FP = 1108896, FN = 148145, TP = 10659\n",
      "\n",
      "LR's TN = 1059341, FP = 1211781, FN = 157598, TP = 1206\n",
      "2022-03-14 06:48:58\n",
      "F1-score for LR: TN = 1059341, FP = 1211781, FN = 157598, TP = 1206\n",
      "\n",
      "MLP's TN = 1265388, FP = 1005734, FN = 157728, TP = 1076\n",
      "2022-03-14 06:49:02\n",
      "F1-score for MLP: TN = 1265388, FP = 1005734, FN = 157728, TP = 1076\n",
      "\n",
      "NB's TN = 1618074, FP = 653048, FN = 158259, TP = 545\n",
      "2022-03-14 06:49:07\n",
      "F1-score for NB: TN = 1618074, FP = 653048, FN = 158259, TP = 545\n",
      "\n",
      "SVM's TN = 682915, FP = 1588207, FN = 11303, TP = 147501\n",
      "2022-03-14 06:49:10\n",
      "F1-score for SVM: TN = 682915, FP = 1588207, FN = 11303, TP = 147501\n",
      "\n",
      "XGB's TN = 1189856, FP = 1081266, FN = 150509, TP = 8295\n",
      "2022-03-14 06:49:15\n",
      "F1-score for XGB: TN = 1189856, FP = 1081266, FN = 150509, TP = 8295\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-03-14 06:50:53\n",
      "Evaluating SCAN_NB15_CIC_us's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's TN = 1955551, FP = 558508, FN = 24998, TP = 11207\n",
      "2022-03-14 06:50:56\n",
      "F1-score for DT: TN = 1955551, FP = 558508, FN = 24998, TP = 11207\n",
      "\n",
      "LR's TN = 2142556, FP = 371503, FN = 33231, TP = 2974\n",
      "2022-03-14 06:50:59\n",
      "F1-score for LR: TN = 2142556, FP = 371503, FN = 33231, TP = 2974\n",
      "\n",
      "MLP's TN = 2175356, FP = 338703, FN = 33234, TP = 2971\n",
      "2022-03-14 06:51:03\n",
      "F1-score for MLP: TN = 2175356, FP = 338703, FN = 33234, TP = 2971\n",
      "\n",
      "NB's TN = 2514049, FP = 10, FN = 36205, TP = 0\n",
      "2022-03-14 06:51:09\n",
      "F1-score for NB: TN = 2514049, FP = 10, FN = 36205, TP = 0\n",
      "\n",
      "SVM's TN = 2142573, FP = 371486, FN = 33231, TP = 2974\n",
      "2022-03-14 06:51:12\n",
      "F1-score for SVM: TN = 2142573, FP = 371486, FN = 33231, TP = 2974\n",
      "\n",
      "XGB's TN = 1955976, FP = 558083, FN = 25016, TP = 11189\n",
      "2022-03-14 06:51:17\n",
      "F1-score for XGB: TN = 1955976, FP = 558083, FN = 25016, TP = 11189\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-03-14 06:56:03\n",
      "Evaluating SCAN_NB15_CIC_us's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's TN = 54867, FP = 34367, FN = 1692490, TP = 1820920\n",
      "2022-03-14 06:56:10\n",
      "F1-score for DT: TN = 54867, FP = 34367, FN = 1692490, TP = 1820920\n",
      "\n",
      "LR's TN = 63776, FP = 25458, FN = 887064, TP = 2626346\n",
      "2022-03-14 06:56:15\n",
      "F1-score for LR: TN = 63776, FP = 25458, FN = 887064, TP = 2626346\n",
      "\n",
      "MLP's TN = 80906, FP = 8328, FN = 2381733, TP = 1131677\n",
      "2022-03-14 06:56:22\n",
      "F1-score for MLP: TN = 80906, FP = 8328, FN = 2381733, TP = 1131677\n",
      "\n",
      "NB's TN = 79806, FP = 9428, FN = 1386332, TP = 2127078\n",
      "2022-03-14 06:56:31\n",
      "F1-score for NB: TN = 79806, FP = 9428, FN = 1386332, TP = 2127078\n",
      "\n",
      "SVM's TN = 14371, FP = 74863, FN = 158029, TP = 3355381\n",
      "2022-03-14 06:56:35\n",
      "F1-score for SVM: TN = 14371, FP = 74863, FN = 158029, TP = 3355381\n",
      "\n",
      "XGB's TN = 56328, FP = 32906, FN = 1660964, TP = 1852446\n",
      "2022-03-14 06:56:43\n",
      "F1-score for XGB: TN = 56328, FP = 32906, FN = 1660964, TP = 1852446\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-03-14 06:59:06\n",
      "Evaluating SCAN_NB15_CIC_us's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's TN = 7845521, FP = 1286832, FN = 78418, TP = 7118\n",
      "2022-03-14 06:59:20\n",
      "F1-score for DT: TN = 7845521, FP = 1286832, FN = 78418, TP = 7118\n",
      "\n",
      "LR's TN = 2220995, FP = 6911358, FN = 52626, TP = 32910\n",
      "2022-03-14 06:59:33\n",
      "F1-score for LR: TN = 2220995, FP = 6911358, FN = 52626, TP = 32910\n",
      "\n",
      "MLP's TN = 923570, FP = 8208783, FN = 4509, TP = 81027\n",
      "2022-03-14 06:59:47\n",
      "F1-score for MLP: TN = 923570, FP = 8208783, FN = 4509, TP = 81027\n",
      "\n",
      "NB's TN = 7983602, FP = 1148751, FN = 78245, TP = 7291\n",
      "2022-03-14 07:07:55\n",
      "F1-score for NB: TN = 7983602, FP = 1148751, FN = 78245, TP = 7291\n",
      "\n",
      "SVM's TN = 3460832, FP = 5671521, FN = 59894, TP = 25642\n",
      "2022-03-14 07:09:30\n",
      "F1-score for SVM: TN = 3460832, FP = 5671521, FN = 59894, TP = 25642\n",
      "\n",
      "XGB's TN = 7867853, FP = 1264500, FN = 80677, TP = 4859\n",
      "2022-03-14 07:09:50\n",
      "F1-score for XGB: TN = 7867853, FP = 1264500, FN = 80677, TP = 4859\n",
      "\n",
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "Using models with undersampled majority class\n",
      "loading models from SCAN_CIC-IDS_CIC_us\n",
      "Models fetched: ['SCAN_CIC-IDS_CIC_us_DT.joblib', 'SCAN_CIC-IDS_CIC_us_LR.joblib', 'SCAN_CIC-IDS_CIC_us_MLP.joblib', 'SCAN_CIC-IDS_CIC_us_NB.joblib', 'SCAN_CIC-IDS_CIC_us_SVM.joblib', 'SCAN_CIC-IDS_CIC_us_XGB.joblib']\n",
      "DT's index of best performance: 1\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "F1-score file for SCAN_CIC-IDS_CIC_us data set not found. Creating..\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-03-14 07:12:40\n",
      "Evaluating SCAN_CIC-IDS_CIC_us's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's TN = 9067778, FP = 64575, FN = 161724, TP = 299\n",
      "2022-03-14 07:12:54\n",
      "F1-score for DT: TN = 9067778, FP = 64575, FN = 161724, TP = 299\n",
      "\n",
      "LR's TN = 9098701, FP = 33652, FN = 162023, TP = 0\n",
      "2022-03-14 07:13:07\n",
      "F1-score for LR: TN = 9098701, FP = 33652, FN = 162023, TP = 0\n",
      "\n",
      "MLP's TN = 8184808, FP = 947545, FN = 145698, TP = 16325\n",
      "2022-03-14 07:13:21\n",
      "F1-score for MLP: TN = 8184808, FP = 947545, FN = 145698, TP = 16325\n",
      "\n",
      "NB's TN = 9132353, FP = 0, FN = 162023, TP = 0\n",
      "2022-03-14 07:18:42\n",
      "F1-score for NB: TN = 9132353, FP = 0, FN = 162023, TP = 0\n",
      "\n",
      "SVM's TN = 9108800, FP = 23553, FN = 162023, TP = 0\n",
      "2022-03-14 07:19:42\n",
      "F1-score for SVM: TN = 9108800, FP = 23553, FN = 162023, TP = 0\n",
      "\n",
      "XGB's TN = 9132353, FP = 0, FN = 162023, TP = 0\n",
      "2022-03-14 07:20:13\n",
      "F1-score for XGB: TN = 9132353, FP = 0, FN = 162023, TP = 0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-03-14 07:20:18\n",
      "Evaluating SCAN_CIC-IDS_CIC_us's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "2022-03-14 07:20:19\n",
      "F1-score for DT: TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "\n",
      "LR's TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "2022-03-14 07:20:19\n",
      "F1-score for LR: TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "\n",
      "MLP's TN = 29573, FP = 7102, FN = 401, TP = 0\n",
      "2022-03-14 07:20:20\n",
      "F1-score for MLP: TN = 29573, FP = 7102, FN = 401, TP = 0\n",
      "\n",
      "NB's TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "2022-03-14 07:20:20\n",
      "F1-score for NB: TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "\n",
      "SVM's TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "2022-03-14 07:20:20\n",
      "F1-score for SVM: TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "\n",
      "XGB's TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "2022-03-14 07:20:21\n",
      "F1-score for XGB: TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-03-14 07:23:12\n",
      "Evaluating SCAN_CIC-IDS_CIC_us's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's TN = 2496550, FP = 17509, FN = 31220, TP = 4985\n",
      "2022-03-14 07:23:15\n",
      "F1-score for DT: TN = 2496550, FP = 17509, FN = 31220, TP = 4985\n",
      "\n",
      "LR's TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "2022-03-14 07:23:18\n",
      "F1-score for LR: TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "\n",
      "MLP's TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "2022-03-14 07:23:22\n",
      "F1-score for MLP: TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "\n",
      "NB's TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "2022-03-14 07:23:28\n",
      "F1-score for NB: TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "\n",
      "SVM's TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "2022-03-14 07:23:31\n",
      "F1-score for SVM: TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "\n",
      "XGB's TN = 2511809, FP = 2250, FN = 36157, TP = 48\n",
      "2022-03-14 07:23:37\n",
      "F1-score for XGB: TN = 2511809, FP = 2250, FN = 36157, TP = 48\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-03-14 07:30:43\n",
      "Evaluating SCAN_CIC-IDS_CIC_us's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's TN = 81395, FP = 7839, FN = 3488897, TP = 24513\n",
      "2022-03-14 07:31:00\n",
      "F1-score for DT: TN = 81395, FP = 7839, FN = 3488897, TP = 24513\n",
      "\n",
      "LR's TN = 87503, FP = 1731, FN = 3503084, TP = 10326\n",
      "2022-03-14 07:31:08\n",
      "F1-score for LR: TN = 87503, FP = 1731, FN = 3503084, TP = 10326\n",
      "\n",
      "MLP's TN = 85521, FP = 3713, FN = 3459439, TP = 53971\n",
      "2022-03-14 07:31:15\n",
      "F1-score for MLP: TN = 85521, FP = 3713, FN = 3459439, TP = 53971\n",
      "\n",
      "NB's TN = 89234, FP = 0, FN = 3513410, TP = 0\n",
      "2022-03-14 07:31:24\n",
      "F1-score for NB: TN = 89234, FP = 0, FN = 3513410, TP = 0\n",
      "\n",
      "SVM's TN = 87293, FP = 1941, FN = 3509178, TP = 4232\n",
      "2022-03-14 07:31:29\n",
      "F1-score for SVM: TN = 87293, FP = 1941, FN = 3509178, TP = 4232\n",
      "\n",
      "XGB's TN = 89170, FP = 64, FN = 3513232, TP = 178\n",
      "2022-03-14 07:31:44\n",
      "F1-score for XGB: TN = 89170, FP = 64, FN = 3513232, TP = 178\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-03-14 07:34:14\n",
      "Evaluating SCAN_CIC-IDS_CIC_us's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's TN = 9067778, FP = 64575, FN = 85536, TP = 0\n",
      "2022-03-14 07:34:29\n",
      "F1-score for DT: TN = 9067778, FP = 64575, FN = 85536, TP = 0\n",
      "\n",
      "LR's TN = 9098701, FP = 33652, FN = 85536, TP = 0\n",
      "2022-03-14 07:34:41\n",
      "F1-score for LR: TN = 9098701, FP = 33652, FN = 85536, TP = 0\n",
      "\n",
      "MLP's TN = 8184808, FP = 947545, FN = 85449, TP = 87\n",
      "2022-03-14 07:34:56\n",
      "F1-score for MLP: TN = 8184808, FP = 947545, FN = 85449, TP = 87\n",
      "\n",
      "NB's TN = 9132353, FP = 0, FN = 85536, TP = 0\n",
      "2022-03-14 07:36:52\n",
      "F1-score for NB: TN = 9132353, FP = 0, FN = 85536, TP = 0\n",
      "\n",
      "SVM's TN = 9108800, FP = 23553, FN = 85536, TP = 0\n",
      "2022-03-14 07:37:18\n",
      "F1-score for SVM: TN = 9108800, FP = 23553, FN = 85536, TP = 0\n",
      "\n",
      "XGB's TN = 9132353, FP = 0, FN = 85536, TP = 0\n",
      "2022-03-14 07:38:04\n",
      "F1-score for XGB: TN = 9132353, FP = 0, FN = 85536, TP = 0\n",
      "\n",
      "ToN-IoT and BoT-IoT only available in CIC dataset type\n",
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "Using models with undersampled majority class\n",
      "loading models from SCAN_ToN-IoT_CIC_us\n",
      "Models fetched: ['SCAN_ToN-IoT_CIC_us_DT.joblib', 'SCAN_ToN-IoT_CIC_us_LR.joblib', 'SCAN_ToN-IoT_CIC_us_MLP.joblib', 'SCAN_ToN-IoT_CIC_us_NB.joblib', 'SCAN_ToN-IoT_CIC_us_SVM.joblib', 'SCAN_ToN-IoT_CIC_us_XGB.joblib']\n",
      "DT's index of best performance: 7\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "F1-score file for SCAN_ToN-IoT_CIC_us data set not found. Creating..\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-03-14 07:40:47\n",
      "Evaluating SCAN_ToN-IoT_CIC_us's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's TN = 8925308, FP = 207045, FN = 162021, TP = 2\n",
      "2022-03-14 07:41:01\n",
      "F1-score for DT: TN = 8925308, FP = 207045, FN = 162021, TP = 2\n",
      "\n",
      "LR's TN = 9050503, FP = 81850, FN = 161915, TP = 108\n",
      "2022-03-14 07:41:15\n",
      "F1-score for LR: TN = 9050503, FP = 81850, FN = 161915, TP = 108\n",
      "\n",
      "MLP's TN = 9046952, FP = 85401, FN = 162001, TP = 22\n",
      "2022-03-14 07:41:31\n",
      "F1-score for MLP: TN = 9046952, FP = 85401, FN = 162001, TP = 22\n",
      "\n",
      "NB's TN = 9132353, FP = 0, FN = 162023, TP = 0\n",
      "2022-03-14 07:44:48\n",
      "F1-score for NB: TN = 9132353, FP = 0, FN = 162023, TP = 0\n",
      "\n",
      "SVM's TN = 8983285, FP = 149068, FN = 161731, TP = 292\n",
      "2022-03-14 07:45:49\n",
      "F1-score for SVM: TN = 8983285, FP = 149068, FN = 161731, TP = 292\n",
      "\n",
      "XGB's TN = 8692629, FP = 439724, FN = 158921, TP = 3102\n",
      "2022-03-14 07:46:16\n",
      "F1-score for XGB: TN = 8692629, FP = 439724, FN = 158921, TP = 3102\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-03-14 07:46:21\n",
      "Evaluating SCAN_ToN-IoT_CIC_us's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's TN = 36672, FP = 3, FN = 401, TP = 0\n",
      "2022-03-14 07:46:22\n",
      "F1-score for DT: TN = 36672, FP = 3, FN = 401, TP = 0\n",
      "\n",
      "LR's TN = 34197, FP = 2478, FN = 396, TP = 5\n",
      "2022-03-14 07:46:22\n",
      "F1-score for LR: TN = 34197, FP = 2478, FN = 396, TP = 5\n",
      "\n",
      "MLP's TN = 11694, FP = 24981, FN = 348, TP = 53\n",
      "2022-03-14 07:46:23\n",
      "F1-score for MLP: TN = 11694, FP = 24981, FN = 348, TP = 53\n",
      "\n",
      "NB's TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "2022-03-14 07:46:23\n",
      "F1-score for NB: TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "\n",
      "SVM's TN = 36670, FP = 5, FN = 397, TP = 4\n",
      "2022-03-14 07:46:23\n",
      "F1-score for SVM: TN = 36670, FP = 5, FN = 397, TP = 4\n",
      "\n",
      "XGB's TN = 25709, FP = 10966, FN = 308, TP = 93\n",
      "2022-03-14 07:46:23\n",
      "F1-score for XGB: TN = 25709, FP = 10966, FN = 308, TP = 93\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-03-14 07:47:09\n",
      "Evaluating SCAN_ToN-IoT_CIC_us's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's TN = 2271051, FP = 71, FN = 158804, TP = 0\n",
      "2022-03-14 07:47:12\n",
      "F1-score for DT: TN = 2271051, FP = 71, FN = 158804, TP = 0\n",
      "\n",
      "LR's TN = 2195347, FP = 75775, FN = 153919, TP = 4885\n",
      "2022-03-14 07:47:15\n",
      "F1-score for LR: TN = 2195347, FP = 75775, FN = 153919, TP = 4885\n",
      "\n",
      "MLP's TN = 1745277, FP = 525845, FN = 158641, TP = 163\n",
      "2022-03-14 07:47:19\n",
      "F1-score for MLP: TN = 1745277, FP = 525845, FN = 158641, TP = 163\n",
      "\n",
      "NB's TN = 2271074, FP = 48, FN = 158804, TP = 0\n",
      "2022-03-14 07:47:25\n",
      "F1-score for NB: TN = 2271074, FP = 48, FN = 158804, TP = 0\n",
      "\n",
      "SVM's TN = 2189582, FP = 81540, FN = 157582, TP = 1222\n",
      "2022-03-14 07:47:28\n",
      "F1-score for SVM: TN = 2189582, FP = 81540, FN = 157582, TP = 1222\n",
      "\n",
      "XGB's TN = 1072085, FP = 1199037, FN = 156186, TP = 2618\n",
      "2022-03-14 07:47:34\n",
      "F1-score for XGB: TN = 1072085, FP = 1199037, FN = 156186, TP = 2618\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-03-14 07:53:09\n",
      "Evaluating SCAN_ToN-IoT_CIC_us's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's TN = 86088, FP = 3146, FN = 3485297, TP = 28113\n",
      "2022-03-14 07:53:29\n",
      "F1-score for DT: TN = 86088, FP = 3146, FN = 3485297, TP = 28113\n",
      "\n",
      "LR's TN = 48765, FP = 40469, FN = 3239350, TP = 274060\n",
      "2022-03-14 07:53:36\n",
      "F1-score for LR: TN = 48765, FP = 40469, FN = 3239350, TP = 274060\n",
      "\n",
      "MLP's TN = 44673, FP = 44561, FN = 3063256, TP = 450154\n",
      "2022-03-14 07:53:45\n",
      "F1-score for MLP: TN = 44673, FP = 44561, FN = 3063256, TP = 450154\n",
      "\n",
      "NB's TN = 89234, FP = 0, FN = 3513410, TP = 0\n",
      "2022-03-14 07:53:55\n",
      "F1-score for NB: TN = 89234, FP = 0, FN = 3513410, TP = 0\n",
      "\n",
      "SVM's TN = 58309, FP = 30925, FN = 3404578, TP = 108832\n",
      "2022-03-14 07:54:00\n",
      "F1-score for SVM: TN = 58309, FP = 30925, FN = 3404578, TP = 108832\n",
      "\n",
      "XGB's TN = 82927, FP = 6307, FN = 1694249, TP = 1819161\n",
      "2022-03-14 07:54:20\n",
      "F1-score for XGB: TN = 82927, FP = 6307, FN = 1694249, TP = 1819161\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-03-14 07:57:12\n",
      "Evaluating SCAN_ToN-IoT_CIC_us's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's TN = 8925308, FP = 207045, FN = 85536, TP = 0\n",
      "2022-03-14 07:57:26\n",
      "F1-score for DT: TN = 8925308, FP = 207045, FN = 85536, TP = 0\n",
      "\n",
      "LR's TN = 9050503, FP = 81850, FN = 85536, TP = 0\n",
      "2022-03-14 07:57:39\n",
      "F1-score for LR: TN = 9050503, FP = 81850, FN = 85536, TP = 0\n",
      "\n",
      "MLP's TN = 9046952, FP = 85401, FN = 85536, TP = 0\n",
      "2022-03-14 07:57:54\n",
      "F1-score for MLP: TN = 9046952, FP = 85401, FN = 85536, TP = 0\n",
      "\n",
      "NB's TN = 9132353, FP = 0, FN = 85536, TP = 0\n",
      "2022-03-14 07:59:41\n",
      "F1-score for NB: TN = 9132353, FP = 0, FN = 85536, TP = 0\n",
      "\n",
      "SVM's TN = 8983285, FP = 149068, FN = 85536, TP = 0\n",
      "2022-03-14 08:00:25\n",
      "F1-score for SVM: TN = 8983285, FP = 149068, FN = 85536, TP = 0\n",
      "\n",
      "XGB's TN = 8692629, FP = 439724, FN = 79629, TP = 5907\n",
      "2022-03-14 08:00:56\n",
      "F1-score for XGB: TN = 8692629, FP = 439724, FN = 79629, TP = 5907\n",
      "\n",
      "ToN-IoT and BoT-IoT only available in CIC dataset type\n",
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "Using models with undersampled majority class\n",
      "loading models from SCAN_BoT-IoT_CIC_us\n",
      "Models fetched: ['SCAN_BoT-IoT_CIC_us_DT.joblib', 'SCAN_BoT-IoT_CIC_us_LR.joblib', 'SCAN_BoT-IoT_CIC_us_MLP.joblib', 'SCAN_BoT-IoT_CIC_us_NB.joblib', 'SCAN_BoT-IoT_CIC_us_SVM.joblib', 'SCAN_BoT-IoT_CIC_us_XGB.joblib']\n",
      "DT's index of best performance: 11\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "F1-score file for SCAN_BoT-IoT_CIC_us data set not found. Creating..\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-03-14 08:03:30\n",
      "Evaluating SCAN_BoT-IoT_CIC_us's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's TN = 6406346, FP = 2726007, FN = 87214, TP = 74809\n",
      "2022-03-14 08:03:44\n",
      "F1-score for DT: TN = 6406346, FP = 2726007, FN = 87214, TP = 74809\n",
      "\n",
      "LR's TN = 4778761, FP = 4353592, FN = 150953, TP = 11070\n",
      "2022-03-14 08:03:57\n",
      "F1-score for LR: TN = 4778761, FP = 4353592, FN = 150953, TP = 11070\n",
      "\n",
      "MLP's TN = 1970568, FP = 7161785, FN = 156638, TP = 5385\n",
      "2022-03-14 08:04:12\n",
      "F1-score for MLP: TN = 1970568, FP = 7161785, FN = 156638, TP = 5385\n",
      "\n",
      "NB's TN = 3177913, FP = 5954440, FN = 161358, TP = 665\n",
      "2022-03-14 08:07:22\n",
      "F1-score for NB: TN = 3177913, FP = 5954440, FN = 161358, TP = 665\n",
      "\n",
      "SVM's TN = 8392161, FP = 740192, FN = 155605, TP = 6418\n",
      "2022-03-14 08:09:31\n",
      "F1-score for SVM: TN = 8392161, FP = 740192, FN = 155605, TP = 6418\n",
      "\n",
      "XGB's TN = 8810960, FP = 321393, FN = 161177, TP = 846\n",
      "2022-03-14 08:10:02\n",
      "F1-score for XGB: TN = 8810960, FP = 321393, FN = 161177, TP = 846\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-03-14 08:10:06\n",
      "Evaluating SCAN_BoT-IoT_CIC_us's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's TN = 26176, FP = 10499, FN = 400, TP = 1\n",
      "2022-03-14 08:10:07\n",
      "F1-score for DT: TN = 26176, FP = 10499, FN = 400, TP = 1\n",
      "\n",
      "LR's TN = 22486, FP = 14189, FN = 152, TP = 249\n",
      "2022-03-14 08:10:07\n",
      "F1-score for LR: TN = 22486, FP = 14189, FN = 152, TP = 249\n",
      "\n",
      "MLP's TN = 34429, FP = 2246, FN = 0, TP = 401\n",
      "2022-03-14 08:10:08\n",
      "F1-score for MLP: TN = 34429, FP = 2246, FN = 0, TP = 401\n",
      "\n",
      "NB's TN = 35530, FP = 1145, FN = 152, TP = 249\n",
      "2022-03-14 08:10:08\n",
      "F1-score for NB: TN = 35530, FP = 1145, FN = 152, TP = 249\n",
      "\n",
      "SVM's TN = 34697, FP = 1978, FN = 259, TP = 142\n",
      "2022-03-14 08:10:08\n",
      "F1-score for SVM: TN = 34697, FP = 1978, FN = 259, TP = 142\n",
      "\n",
      "XGB's TN = 36411, FP = 264, FN = 401, TP = 0\n",
      "2022-03-14 08:10:08\n",
      "F1-score for XGB: TN = 36411, FP = 264, FN = 401, TP = 0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-03-14 08:10:54\n",
      "Evaluating SCAN_BoT-IoT_CIC_us's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's TN = 1815973, FP = 455149, FN = 82368, TP = 76436\n",
      "2022-03-14 08:10:57\n",
      "F1-score for DT: TN = 1815973, FP = 455149, FN = 82368, TP = 76436\n",
      "\n",
      "LR's TN = 1706149, FP = 564973, FN = 157740, TP = 1064\n",
      "2022-03-14 08:11:00\n",
      "F1-score for LR: TN = 1706149, FP = 564973, FN = 157740, TP = 1064\n",
      "\n",
      "MLP's TN = 1604417, FP = 666705, FN = 877, TP = 157927\n",
      "2022-03-14 08:11:04\n",
      "F1-score for MLP: TN = 1604417, FP = 666705, FN = 877, TP = 157927\n",
      "\n",
      "NB's TN = 1704152, FP = 566970, FN = 158628, TP = 176\n",
      "2022-03-14 08:11:09\n",
      "F1-score for NB: TN = 1704152, FP = 566970, FN = 158628, TP = 176\n",
      "\n",
      "SVM's TN = 1680605, FP = 590517, FN = 157315, TP = 1489\n",
      "2022-03-14 08:11:12\n",
      "F1-score for SVM: TN = 1680605, FP = 590517, FN = 157315, TP = 1489\n",
      "\n",
      "XGB's TN = 2097611, FP = 173511, FN = 124049, TP = 34755\n",
      "2022-03-14 08:11:18\n",
      "F1-score for XGB: TN = 2097611, FP = 173511, FN = 124049, TP = 34755\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-03-14 08:13:16\n",
      "Evaluating SCAN_BoT-IoT_CIC_us's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's TN = 2225701, FP = 288358, FN = 24457, TP = 11748\n",
      "2022-03-14 08:13:19\n",
      "F1-score for DT: TN = 2225701, FP = 288358, FN = 24457, TP = 11748\n",
      "\n",
      "LR's TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "2022-03-14 08:13:22\n",
      "F1-score for LR: TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "\n",
      "MLP's TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "2022-03-14 08:13:26\n",
      "F1-score for MLP: TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "\n",
      "NB's TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "2022-03-14 08:13:31\n",
      "F1-score for NB: TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "\n",
      "SVM's TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "2022-03-14 08:13:34\n",
      "F1-score for SVM: TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "\n",
      "XGB's TN = 2485615, FP = 28444, FN = 31229, TP = 4976\n",
      "2022-03-14 08:13:41\n",
      "F1-score for XGB: TN = 2485615, FP = 28444, FN = 31229, TP = 4976\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-03-14 08:16:41\n",
      "Evaluating SCAN_BoT-IoT_CIC_us's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's TN = 6406346, FP = 2726007, FN = 54325, TP = 31211\n",
      "2022-03-14 08:16:56\n",
      "F1-score for DT: TN = 6406346, FP = 2726007, FN = 54325, TP = 31211\n",
      "\n",
      "LR's TN = 4778761, FP = 4353592, FN = 62672, TP = 22864\n",
      "2022-03-14 08:17:08\n",
      "F1-score for LR: TN = 4778761, FP = 4353592, FN = 62672, TP = 22864\n",
      "\n",
      "MLP's TN = 1970568, FP = 7161785, FN = 47208, TP = 38328\n",
      "2022-03-14 08:17:23\n",
      "F1-score for MLP: TN = 1970568, FP = 7161785, FN = 47208, TP = 38328\n",
      "\n",
      "NB's TN = 3177913, FP = 5954440, FN = 52133, TP = 33403\n",
      "2022-03-14 08:18:32\n",
      "F1-score for NB: TN = 3177913, FP = 5954440, FN = 52133, TP = 33403\n",
      "\n",
      "SVM's TN = 8392161, FP = 740192, FN = 83801, TP = 1735\n",
      "2022-03-14 08:19:01\n",
      "F1-score for SVM: TN = 8392161, FP = 740192, FN = 83801, TP = 1735\n",
      "\n",
      "XGB's TN = 8810960, FP = 321393, FN = 84258, TP = 1278\n",
      "2022-03-14 08:19:36\n",
      "F1-score for XGB: TN = 8810960, FP = 321393, FN = 84258, TP = 1278\n",
      "\n",
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "Using models with undersampled majority class\n",
      "loading models from SCAN_internet_CIC_us\n",
      "Models fetched: ['SCAN_internet_CIC_us_DT.joblib', 'SCAN_internet_CIC_us_LR.joblib', 'SCAN_internet_CIC_us_MLP.joblib', 'SCAN_internet_CIC_us_NB.joblib', 'SCAN_internet_CIC_us_SVM.joblib', 'SCAN_internet_CIC_us_XGB.joblib']\n",
      "DT's index of best performance: 2\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "F1-score file for SCAN_internet_CIC_us data set not found. Creating..\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-03-14 08:22:15\n",
      "Evaluating SCAN_internet_CIC_us's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's TN = 8740858, FP = 391495, FN = 138591, TP = 23432\n",
      "2022-03-14 08:22:30\n",
      "F1-score for DT: TN = 8740858, FP = 391495, FN = 138591, TP = 23432\n",
      "\n",
      "LR's TN = 7552959, FP = 1579394, FN = 158844, TP = 3179\n",
      "2022-03-14 08:22:43\n",
      "F1-score for LR: TN = 7552959, FP = 1579394, FN = 158844, TP = 3179\n",
      "\n",
      "MLP's TN = 8168344, FP = 964009, FN = 141798, TP = 20225\n",
      "2022-03-14 08:22:59\n",
      "F1-score for MLP: TN = 8168344, FP = 964009, FN = 141798, TP = 20225\n",
      "\n",
      "NB's TN = 2193534, FP = 6938819, FN = 156424, TP = 5599\n",
      "2022-03-14 08:31:07\n",
      "F1-score for NB: TN = 2193534, FP = 6938819, FN = 156424, TP = 5599\n",
      "\n",
      "SVM's TN = 8072250, FP = 1060103, FN = 156908, TP = 5115\n",
      "2022-03-14 08:33:19\n",
      "F1-score for SVM: TN = 8072250, FP = 1060103, FN = 156908, TP = 5115\n",
      "\n",
      "XGB's TN = 8850025, FP = 282328, FN = 160206, TP = 1817\n",
      "2022-03-14 08:33:53\n",
      "F1-score for XGB: TN = 8850025, FP = 282328, FN = 160206, TP = 1817\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-03-14 08:33:58\n",
      "Evaluating SCAN_internet_CIC_us's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's TN = 11801, FP = 24874, FN = 153, TP = 248\n",
      "2022-03-14 08:33:58\n",
      "F1-score for DT: TN = 11801, FP = 24874, FN = 153, TP = 248\n",
      "\n",
      "LR's TN = 11781, FP = 24894, FN = 309, TP = 92\n",
      "2022-03-14 08:33:58\n",
      "F1-score for LR: TN = 11781, FP = 24894, FN = 309, TP = 92\n",
      "\n",
      "MLP's TN = 1201, FP = 35474, FN = 295, TP = 106\n",
      "2022-03-14 08:34:00\n",
      "F1-score for MLP: TN = 1201, FP = 35474, FN = 295, TP = 106\n",
      "\n",
      "NB's TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "2022-03-14 08:34:00\n",
      "F1-score for NB: TN = 36675, FP = 0, FN = 401, TP = 0\n",
      "\n",
      "SVM's TN = 8391, FP = 28284, FN = 259, TP = 142\n",
      "2022-03-14 08:34:00\n",
      "F1-score for SVM: TN = 8391, FP = 28284, FN = 259, TP = 142\n",
      "\n",
      "XGB's TN = 36661, FP = 14, FN = 401, TP = 0\n",
      "2022-03-14 08:34:00\n",
      "F1-score for XGB: TN = 36661, FP = 14, FN = 401, TP = 0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-03-14 08:35:27\n",
      "Evaluating SCAN_internet_CIC_us's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's TN = 1763458, FP = 507664, FN = 92488, TP = 66316\n",
      "2022-03-14 08:35:31\n",
      "F1-score for DT: TN = 1763458, FP = 507664, FN = 92488, TP = 66316\n",
      "\n",
      "LR's TN = 1368523, FP = 902599, FN = 158433, TP = 371\n",
      "2022-03-14 08:35:34\n",
      "F1-score for LR: TN = 1368523, FP = 902599, FN = 158433, TP = 371\n",
      "\n",
      "MLP's TN = 770257, FP = 1500865, FN = 135428, TP = 23376\n",
      "2022-03-14 08:35:37\n",
      "F1-score for MLP: TN = 770257, FP = 1500865, FN = 135428, TP = 23376\n",
      "\n",
      "NB's TN = 2264345, FP = 6777, FN = 158804, TP = 0\n",
      "2022-03-14 08:35:43\n",
      "F1-score for NB: TN = 2264345, FP = 6777, FN = 158804, TP = 0\n",
      "\n",
      "SVM's TN = 1172797, FP = 1098325, FN = 158678, TP = 126\n",
      "2022-03-14 08:35:46\n",
      "F1-score for SVM: TN = 1172797, FP = 1098325, FN = 158678, TP = 126\n",
      "\n",
      "XGB's TN = 2260513, FP = 10609, FN = 158573, TP = 231\n",
      "2022-03-14 08:35:51\n",
      "F1-score for XGB: TN = 2260513, FP = 10609, FN = 158573, TP = 231\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-03-14 08:37:50\n",
      "Evaluating SCAN_internet_CIC_us's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's TN = 1987184, FP = 526875, FN = 18435, TP = 17770\n",
      "2022-03-14 08:37:53\n",
      "F1-score for DT: TN = 1987184, FP = 526875, FN = 18435, TP = 17770\n",
      "\n",
      "LR's TN = 7, FP = 2514052, FN = 0, TP = 36205\n",
      "2022-03-14 08:37:56\n",
      "F1-score for LR: TN = 7, FP = 2514052, FN = 0, TP = 36205\n",
      "\n",
      "MLP's TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "2022-03-14 08:37:59\n",
      "F1-score for MLP: TN = 0, FP = 2514059, FN = 0, TP = 36205\n",
      "\n",
      "NB's TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "2022-03-14 08:38:05\n",
      "F1-score for NB: TN = 2514059, FP = 0, FN = 36205, TP = 0\n",
      "\n",
      "SVM's TN = 81, FP = 2513978, FN = 0, TP = 36205\n",
      "2022-03-14 08:38:08\n",
      "F1-score for SVM: TN = 81, FP = 2513978, FN = 0, TP = 36205\n",
      "\n",
      "XGB's TN = 2512711, FP = 1348, FN = 36205, TP = 0\n",
      "2022-03-14 08:38:14\n",
      "F1-score for XGB: TN = 2512711, FP = 1348, FN = 36205, TP = 0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-03-14 08:43:11\n",
      "Evaluating SCAN_internet_CIC_us's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's TN = 86983, FP = 2251, FN = 2353942, TP = 1159468\n",
      "2022-03-14 08:43:29\n",
      "F1-score for DT: TN = 86983, FP = 2251, FN = 2353942, TP = 1159468\n",
      "\n",
      "LR's TN = 82449, FP = 6785, FN = 2301748, TP = 1211662\n",
      "2022-03-14 08:43:40\n",
      "F1-score for LR: TN = 82449, FP = 6785, FN = 2301748, TP = 1211662\n",
      "\n",
      "MLP's TN = 57745, FP = 31489, FN = 1941544, TP = 1571866\n",
      "2022-03-14 08:43:47\n",
      "F1-score for MLP: TN = 57745, FP = 31489, FN = 1941544, TP = 1571866\n",
      "\n",
      "NB's TN = 89233, FP = 1, FN = 3513407, TP = 3\n",
      "2022-03-14 08:43:56\n",
      "F1-score for NB: TN = 89233, FP = 1, FN = 3513407, TP = 3\n",
      "\n",
      "SVM's TN = 33559, FP = 55675, FN = 2381938, TP = 1131472\n",
      "2022-03-14 08:44:01\n",
      "F1-score for SVM: TN = 33559, FP = 55675, FN = 2381938, TP = 1131472\n",
      "\n",
      "XGB's TN = 88666, FP = 568, FN = 3513410, TP = 0\n",
      "2022-03-14 08:44:13\n",
      "F1-score for XGB: TN = 88666, FP = 568, FN = 3513410, TP = 0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 0 $TYPE_OF_TEST\n",
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 1 $TYPE_OF_TEST\n",
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 2 $TYPE_OF_TEST\n",
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 3 $TYPE_OF_TEST\n",
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 4 $TYPE_OF_TEST\n",
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 5 $TYPE_OF_TEST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "21bdd177-21bc-4f58-ae3f-a26df2693f2c",
   "metadata": {},
   "source": [
    "# Run Individually"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1bc34b29-138e-4ce8-9c5c-6513ae70a2c7",
   "metadata": {
    "tags": []
   },
   "source": [
    "## Test AB-TRAP data set models for scanning attack detection on all other data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "8dfe851d-7f8d-4f15-b329-6884f4c88aa7",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "loading models from SCAN_AB-TRAP_CIC\n",
      "Models fetched: ['SCAN_AB-TRAP_CIC_DT.joblib', 'SCAN_AB-TRAP_CIC_LR.joblib', 'SCAN_AB-TRAP_CIC_MLP.joblib', 'SCAN_AB-TRAP_CIC_NB.joblib', 'SCAN_AB-TRAP_CIC_SVM.joblib', 'SCAN_AB-TRAP_CIC_XGB.joblib']\n",
      "DT's index of best performance: 5\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "DT\n",
      "F1-score file for SCAN_AB-TRAP_CIC data set not found. Creating..\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-01-14 15:40:38\n",
      "Evaluating SCAN_AB-TRAP_CIC's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's scoreA = 0.554954954954955, scoreB = 0.0\n",
      "2022-01-14 15:40:38\n",
      "F1-score for DT: Attack 0.554954954954955, Benign 0.0\n",
      "\n",
      "LR's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 15:40:38\n",
      "F1-score for LR: Attack 0.0, Benign 0.0\n",
      "\n",
      "MLP's scoreA = 0.39599999999999996, scoreB = 0.0\n",
      "2022-01-14 15:40:38\n",
      "F1-score for MLP: Attack 0.39599999999999996, Benign 0.0\n",
      "\n",
      "NB's scoreA = 0.3765182186234818, scoreB = 0.0\n",
      "2022-01-14 15:40:38\n",
      "F1-score for NB: Attack 0.3765182186234818, Benign 0.0\n",
      "\n",
      "SVM's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 15:40:38\n",
      "F1-score for SVM: Attack 0.0, Benign 0.0\n",
      "\n",
      "XGB's scoreA = 0.5497287522603979, scoreB = 0.0\n",
      "2022-01-14 15:40:38\n",
      "F1-score for XGB: Attack 0.5497287522603979, Benign 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-01-14 15:41:23\n",
      "Evaluating SCAN_AB-TRAP_CIC's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's scoreA = 0.00802789694187301, scoreB = 0.0\n",
      "2022-01-14 15:41:27\n",
      "F1-score for DT: Attack 0.00802789694187301, Benign 0.0\n",
      "\n",
      "LR's scoreA = 0.99569005043558, scoreB = 0.0\n",
      "2022-01-14 15:41:29\n",
      "F1-score for LR: Attack 0.99569005043558, Benign 0.0\n",
      "\n",
      "MLP's scoreA = 0.9991365564358057, scoreB = 0.0\n",
      "2022-01-14 15:41:32\n",
      "F1-score for MLP: Attack 0.9991365564358057, Benign 0.0\n",
      "\n",
      "NB's scoreA = 0.9957122800804422, scoreB = 0.0\n",
      "2022-01-14 15:41:38\n",
      "F1-score for NB: Attack 0.9957122800804422, Benign 0.0\n",
      "\n",
      "SVM's scoreA = 0.9986758641562733, scoreB = 0.0\n",
      "2022-01-14 15:41:41\n",
      "F1-score for SVM: Attack 0.9986758641562733, Benign 0.0\n",
      "\n",
      "XGB's scoreA = 0.3588439855937414, scoreB = 0.0\n",
      "2022-01-14 15:41:46\n",
      "F1-score for XGB: Attack 0.3588439855937414, Benign 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-01-14 15:42:34\n",
      "Evaluating SCAN_AB-TRAP_CIC's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's scoreA = 0.6124365239053369, scoreB = 0.0\n",
      "2022-01-14 15:42:37\n",
      "F1-score for DT: Attack 0.6124365239053369, Benign 0.0\n",
      "\n",
      "LR's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 15:42:40\n",
      "F1-score for LR: Attack 0.0, Benign 0.0\n",
      "\n",
      "MLP's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 15:42:44\n",
      "F1-score for MLP: Attack 0.0, Benign 0.0\n",
      "\n",
      "NB's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 15:42:49\n",
      "F1-score for NB: Attack 0.0, Benign 0.0\n",
      "\n",
      "SVM's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 15:42:52\n",
      "F1-score for SVM: Attack 0.0, Benign 0.0\n",
      "\n",
      "XGB's scoreA = 0.6257875958399757, scoreB = 0.0\n",
      "2022-01-14 15:42:58\n",
      "F1-score for XGB: Attack 0.6257875958399757, Benign 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-01-14 15:46:11\n",
      "Evaluating SCAN_AB-TRAP_CIC's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's scoreA = 0.07646382197568403, scoreB = 0.0\n",
      "2022-01-14 15:46:18\n",
      "F1-score for DT: Attack 0.07646382197568403, Benign 0.0\n",
      "\n",
      "LR's scoreA = 0.3810640232200118, scoreB = 0.0\n",
      "2022-01-14 15:46:23\n",
      "F1-score for LR: Attack 0.3810640232200118, Benign 0.0\n",
      "\n",
      "MLP's scoreA = 0.7930901222536386, scoreB = 0.0\n",
      "2022-01-14 15:46:30\n",
      "F1-score for MLP: Attack 0.7930901222536386, Benign 0.0\n",
      "\n",
      "NB's scoreA = 0.3815875358549615, scoreB = 0.0\n",
      "2022-01-14 15:46:39\n",
      "F1-score for NB: Attack 0.3815875358549615, Benign 0.0\n",
      "\n",
      "SVM's scoreA = 0.6836127223181598, scoreB = 0.0\n",
      "2022-01-14 15:46:44\n",
      "F1-score for SVM: Attack 0.6836127223181598, Benign 0.0\n",
      "\n",
      "XGB's scoreA = 0.2147524623683451, scoreB = 0.0\n",
      "2022-01-14 15:46:53\n",
      "F1-score for XGB: Attack 0.2147524623683451, Benign 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-01-14 15:48:54\n",
      "Evaluating SCAN_AB-TRAP_CIC's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's scoreA = 0.07294928695338726, scoreB = 0.0\n",
      "2022-01-14 15:49:18\n",
      "F1-score for DT: Attack 0.07294928695338726, Benign 0.0\n",
      "\n",
      "LR's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 15:49:36\n",
      "F1-score for LR: Attack 0.0, Benign 0.0\n",
      "\n",
      "MLP's scoreA = 0.06842354854007181, scoreB = 0.0\n",
      "2022-01-14 15:50:15\n",
      "F1-score for MLP: Attack 0.06842354854007181, Benign 0.0\n",
      "\n",
      "NB's scoreA = 0.9750459836915168, scoreB = 0.0\n",
      "2022-01-14 16:02:54\n",
      "F1-score for NB: Attack 0.9750459836915168, Benign 0.0\n",
      "\n",
      "SVM's scoreA = 0.0, scoreB = 0.0\n",
      "2022-01-14 16:03:52\n",
      "F1-score for SVM: Attack 0.0, Benign 0.0\n",
      "\n",
      "XGB's scoreA = 0.06842354854007181, scoreB = 0.0\n",
      "2022-01-14 16:04:54\n",
      "F1-score for XGB: Attack 0.06842354854007181, Benign 0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 0 $TYPE_OF_TEST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e3471cd9-f3b6-45b3-afd0-b9af7329615a",
   "metadata": {},
   "source": [
    "## Test UNSW-NB15 data set models for scanning attack detection on all other data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "546232e8-e9d1-4d35-bb6b-39c8b7a51147",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "loading models from SCAN_NB15_CIC\n",
      "Models fetched: ['SCAN_NB15_CIC_DT.joblib', 'SCAN_NB15_CIC_LR.joblib', 'SCAN_NB15_CIC_MLP.joblib', 'SCAN_NB15_CIC_NB.joblib', 'SCAN_NB15_CIC_SVM.joblib', 'SCAN_NB15_CIC_XGB.joblib']\n",
      "DT's index of best performance: 10\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "DT\n",
      "Found F1-score file for SCAN_NB15_CIC data set\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-01-12 15:33:45\n",
      "Evaluating SCAN_NB15_CIC's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-12 15:34:04\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-12 15:34:19\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 15:34:29\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 15:53:24\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-12 15:54:34\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-12 15:55:43\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-01-12 15:58:31\n",
      "Evaluating SCAN_NB15_CIC's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's score = 0.014299354455852313\n",
      "2022-01-12 15:58:34\n",
      "F1-score for DT: 0.014299354455852313\n",
      "\n",
      "LR's score = 0.0431107766341965\n",
      "2022-01-12 15:58:36\n",
      "F1-score for LR: 0.0431107766341965\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 15:58:38\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0013315058720630524\n",
      "2022-01-12 15:58:43\n",
      "F1-score for NB: 0.0013315058720630524\n",
      "\n",
      "SVM's score = 0.16840477615075194\n",
      "2022-01-12 15:58:46\n",
      "F1-score for SVM: 0.16840477615075194\n",
      "\n",
      "XGB's score = 0.02211697545246279\n",
      "2022-01-12 15:58:49\n",
      "F1-score for XGB: 0.02211697545246279\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-01-12 16:01:31\n",
      "Evaluating SCAN_NB15_CIC's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's score = 0.03708545033874855\n",
      "2022-01-12 16:01:33\n",
      "F1-score for DT: 0.03708545033874855\n",
      "\n",
      "LR's score = 0.014483613403429999\n",
      "2022-01-12 16:01:36\n",
      "F1-score for LR: 0.014483613403429999\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 16:01:38\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 16:01:44\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.014483401797519705\n",
      "2022-01-12 16:01:46\n",
      "F1-score for SVM: 0.014483401797519705\n",
      "\n",
      "XGB's score = 0.03698135044144481\n",
      "2022-01-12 16:01:50\n",
      "F1-score for XGB: 0.03698135044144481\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-01-12 16:06:19\n",
      "Evaluating SCAN_NB15_CIC's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's score = 0.509953005735666\n",
      "2022-01-12 16:06:24\n",
      "F1-score for DT: 0.509953005735666\n",
      "\n",
      "LR's score = 0.8456078976593452\n",
      "2022-01-12 16:06:28\n",
      "F1-score for LR: 0.8456078976593452\n",
      "\n",
      "MLP's score = 0.007728240825797319\n",
      "2022-01-12 16:06:32\n",
      "F1-score for MLP: 0.007728240825797319\n",
      "\n",
      "NB's score = 0.7423881572594456\n",
      "2022-01-12 16:06:40\n",
      "F1-score for NB: 0.7423881572594456\n",
      "\n",
      "SVM's score = 0.9410032731343422\n",
      "2022-01-12 16:06:43\n",
      "F1-score for SVM: 0.9410032731343422\n",
      "\n",
      "XGB's score = 0.5037359691457842\n",
      "2022-01-12 16:06:50\n",
      "F1-score for XGB: 0.5037359691457842\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-01-12 16:10:27\n",
      "Evaluating SCAN_NB15_CIC's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's score = 0.006921532512910294\n",
      "2022-01-12 16:10:37\n",
      "F1-score for DT: 0.006921532512910294\n",
      "\n",
      "LR's score = 0.019761258336006594\n",
      "2022-01-12 16:10:47\n",
      "F1-score for LR: 0.019761258336006594\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 16:10:57\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.011722430856860434\n",
      "2022-01-12 16:12:17\n",
      "F1-score for NB: 0.011722430856860434\n",
      "\n",
      "SVM's score = 0.002302114108122626\n",
      "2022-01-12 16:14:52\n",
      "F1-score for SVM: 0.002302114108122626\n",
      "\n",
      "XGB's score = 0.006385274445682024\n",
      "2022-01-12 16:15:08\n",
      "F1-score for XGB: 0.006385274445682024\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 1 $TYPE_OF_TEST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b6cd5b4c-6bd3-4572-9336-0fc1fc4138d8",
   "metadata": {},
   "source": [
    "## Test CIC-IDS data set models for scanning attack detection on all other data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "90752a84-4895-4b1a-aed3-525631406577",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "loading models from SCAN_CIC-IDS_CIC\n",
      "Models fetched: ['SCAN_CIC-IDS_CIC_DT.joblib', 'SCAN_CIC-IDS_CIC_LR.joblib', 'SCAN_CIC-IDS_CIC_MLP.joblib', 'SCAN_CIC-IDS_CIC_NB.joblib', 'SCAN_CIC-IDS_CIC_SVM.joblib', 'SCAN_CIC-IDS_CIC_XGB.joblib']\n",
      "DT's index of best performance: 4\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "Found F1-score file for SCAN_CIC-IDS_CIC data set\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-01-12 16:36:06\n",
      "Evaluating SCAN_CIC-IDS_CIC's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-12 16:36:17\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-12 16:36:26\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 16:36:37\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 16:40:22\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-12 16:45:22\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-12 16:45:51\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-01-12 16:46:04\n",
      "Evaluating SCAN_CIC-IDS_CIC's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-12 16:46:04\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-12 16:46:04\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 16:46:05\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 16:46:05\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-12 16:46:05\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-12 16:46:05\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-01-12 16:48:36\n",
      "Evaluating SCAN_CIC-IDS_CIC's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's score = 0.007180650037792895\n",
      "2022-01-12 16:48:38\n",
      "F1-score for DT: 0.007180650037792895\n",
      "\n",
      "LR's score = 0.02799569606285635\n",
      "2022-01-12 16:48:41\n",
      "F1-score for LR: 0.02799569606285635\n",
      "\n",
      "MLP's score = 0.02799569606285635\n",
      "2022-01-12 16:48:43\n",
      "F1-score for MLP: 0.02799569606285635\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 16:48:49\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.02799569606285635\n",
      "2022-01-12 16:48:51\n",
      "F1-score for SVM: 0.02799569606285635\n",
      "\n",
      "XGB's score = 0.0017077537529265941\n",
      "2022-01-12 16:48:56\n",
      "F1-score for XGB: 0.0017077537529265941\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-01-12 16:54:20\n",
      "Evaluating SCAN_CIC-IDS_CIC's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's score = 0.00121703969312401\n",
      "2022-01-12 16:54:26\n",
      "F1-score for DT: 0.00121703969312401\n",
      "\n",
      "LR's score = 0.0013689390182312876\n",
      "2022-01-12 16:54:30\n",
      "F1-score for LR: 0.0013689390182312876\n",
      "\n",
      "MLP's score = 0.00047512669096735796\n",
      "2022-01-12 16:54:35\n",
      "F1-score for MLP: 0.00047512669096735796\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 16:54:43\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0024682708722729486\n",
      "2022-01-12 16:54:46\n",
      "F1-score for SVM: 0.0024682708722729486\n",
      "\n",
      "XGB's score = 5.692457579094563e-06\n",
      "2022-01-12 16:54:54\n",
      "F1-score for XGB: 5.692457579094563e-06\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-01-12 16:58:00\n",
      "Evaluating SCAN_CIC-IDS_CIC's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-12 16:58:11\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-12 16:58:21\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 16:58:32\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 17:05:07\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-12 17:07:55\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-12 17:08:14\n",
      "F1-score for XGB: 0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 2 $TYPE_OF_TEST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f4b95c59-66d2-4052-ab60-a8e338982e47",
   "metadata": {},
   "source": [
    "## Test CIC-ToN-IoT data set models for scanning attack detection on all other data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "02f0663b-39c7-4444-a5e8-654cf150c670",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ToN-IoT and BoT-IoT only available in CIC dataset type\n",
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "loading models from SCAN_ToN-IoT_CIC\n",
      "Models fetched: ['SCAN_ToN-IoT_CIC_DT.joblib', 'SCAN_ToN-IoT_CIC_LR.joblib', 'SCAN_ToN-IoT_CIC_MLP.joblib', 'SCAN_ToN-IoT_CIC_NB.joblib', 'SCAN_ToN-IoT_CIC_SVM.joblib', 'SCAN_ToN-IoT_CIC_XGB.joblib']\n",
      "DT's index of best performance: 5\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "Found F1-score file for SCAN_ToN-IoT_CIC data set\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-01-10 13:03:10\n",
      "Evaluating SCAN_ToN-IoT_CIC's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-10 13:03:21\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-10 13:03:31\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-10 13:03:43\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-10 13:06:42\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-10 13:07:08\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-01-10 13:07:20\n",
      "Evaluating SCAN_ToN-IoT_CIC's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's score = 0.07829787234042553\n",
      "2022-01-10 13:07:20\n",
      "F1-score for DT: 0.07829787234042553\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-10 13:07:20\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0208285995607059\n",
      "2022-01-10 13:07:23\n",
      "F1-score for MLP: 0.0208285995607059\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 13:07:23\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-10 13:07:23\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-10 13:07:23\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-01-10 13:09:14\n",
      "Evaluating SCAN_ToN-IoT_CIC's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's score = 0.006131455634829251\n",
      "2022-01-10 13:09:17\n",
      "F1-score for DT: 0.006131455634829251\n",
      "\n",
      "LR's score = 0.010066737595512139\n",
      "2022-01-10 13:09:19\n",
      "F1-score for LR: 0.010066737595512139\n",
      "\n",
      "MLP's score = 0.14741448426557247\n",
      "2022-01-10 13:09:21\n",
      "F1-score for MLP: 0.14741448426557247\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 13:09:26\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-10 13:09:28\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.011257110987172336\n",
      "2022-01-10 13:09:33\n",
      "F1-score for XGB: 0.011257110987172336\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-01-10 13:14:46\n",
      "Evaluating SCAN_ToN-IoT_CIC's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's score = 0.18476216417548563\n",
      "2022-01-10 13:14:52\n",
      "F1-score for DT: 0.18476216417548563\n",
      "\n",
      "LR's score = 0.6289935945600289\n",
      "2022-01-10 13:14:56\n",
      "F1-score for LR: 0.6289935945600289\n",
      "\n",
      "MLP's score = 0.35147221082218316\n",
      "2022-01-10 13:15:00\n",
      "F1-score for MLP: 0.35147221082218316\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 13:15:08\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.02880960316399169\n",
      "2022-01-10 13:15:11\n",
      "F1-score for SVM: 0.02880960316399169\n",
      "\n",
      "XGB's score = 0.11306428656181763\n",
      "2022-01-10 13:15:20\n",
      "F1-score for XGB: 0.11306428656181763\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-01-10 13:18:55\n",
      "Evaluating SCAN_ToN-IoT_CIC's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's score = 0.009569934134056567\n",
      "2022-01-10 13:19:06\n",
      "F1-score for DT: 0.009569934134056567\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-10 13:19:15\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0032110841170047586\n",
      "2022-01-10 13:19:26\n",
      "F1-score for MLP: 0.0032110841170047586\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 13:23:35\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-10 13:24:58\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.04421589212130739\n",
      "2022-01-10 13:25:17\n",
      "F1-score for XGB: 0.04421589212130739\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 3 $TYPE_OF_TEST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a6c7ccad-049d-4cf3-8886-d0b81c320062",
   "metadata": {},
   "source": [
    "## Test CIC-BoT-IoT data set models for scanning attack detection on all other data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "f694bf0a-d4f2-442a-8209-50d9d75e2aaa",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ToN-IoT and BoT-IoT only available in CIC dataset type\n",
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "loading models from SCAN_BoT-IoT_CIC\n",
      "Models fetched: ['SCAN_BoT-IoT_CIC_DT.joblib', 'SCAN_BoT-IoT_CIC_LR.joblib', 'SCAN_BoT-IoT_CIC_MLP.joblib', 'SCAN_BoT-IoT_CIC_NB.joblib', 'SCAN_BoT-IoT_CIC_SVM.joblib', 'SCAN_BoT-IoT_CIC_XGB.joblib']\n",
      "DT's index of best performance: 5\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "XGB\n",
      "Found F1-score file for SCAN_BoT-IoT_CIC data set\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-01-12 17:16:21\n",
      "Evaluating SCAN_BoT-IoT_CIC's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-12 17:16:32\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-12 17:16:42\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-12 17:16:53\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 17:19:26\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-12 17:20:34\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-12 17:21:01\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-01-12 17:21:07\n",
      "Evaluating SCAN_BoT-IoT_CIC's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's score = 0.0008385744234800839\n",
      "2022-01-12 17:21:08\n",
      "F1-score for DT: 0.0008385744234800839\n",
      "\n",
      "LR's score = 0.019863147605083085\n",
      "2022-01-12 17:21:08\n",
      "F1-score for LR: 0.019863147605083085\n",
      "\n",
      "MLP's score = 0.07200116601078561\n",
      "2022-01-12 17:21:10\n",
      "F1-score for MLP: 0.07200116601078561\n",
      "\n",
      "NB's score = 0.2775919732441472\n",
      "2022-01-12 17:21:10\n",
      "F1-score for NB: 0.2775919732441472\n",
      "\n",
      "SVM's score = 0.1866469937292512\n",
      "2022-01-12 17:21:10\n",
      "F1-score for SVM: 0.1866469937292512\n",
      "\n",
      "XGB's score = 0.16512764801738186\n",
      "2022-01-12 17:21:10\n",
      "F1-score for XGB: 0.16512764801738186\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-01-12 17:23:15\n",
      "Evaluating SCAN_BoT-IoT_CIC's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's score = 0.2085202335837017\n",
      "2022-01-12 17:23:17\n",
      "F1-score for DT: 0.2085202335837017\n",
      "\n",
      "LR's score = 0.009897112458286079\n",
      "2022-01-12 17:23:19\n",
      "F1-score for LR: 0.009897112458286079\n",
      "\n",
      "MLP's score = 0.000260551663866641\n",
      "2022-01-12 17:23:22\n",
      "F1-score for MLP: 0.000260551663866641\n",
      "\n",
      "NB's score = 0.0004816164800402806\n",
      "2022-01-12 17:23:26\n",
      "F1-score for NB: 0.0004816164800402806\n",
      "\n",
      "SVM's score = 0.2384034365868089\n",
      "2022-01-12 17:23:28\n",
      "F1-score for SVM: 0.2384034365868089\n",
      "\n",
      "XGB's score = 0.33834931372438737\n",
      "2022-01-12 17:23:33\n",
      "F1-score for XGB: 0.33834931372438737\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-01-12 17:25:33\n",
      "Evaluating SCAN_BoT-IoT_CIC's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's score = 0.09531918010125146\n",
      "2022-01-12 17:25:35\n",
      "F1-score for DT: 0.09531918010125146\n",
      "\n",
      "LR's score = 0.030046003450258766\n",
      "2022-01-12 17:25:38\n",
      "F1-score for LR: 0.030046003450258766\n",
      "\n",
      "MLP's score = 0.02799569606285635\n",
      "2022-01-12 17:25:40\n",
      "F1-score for MLP: 0.02799569606285635\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-12 17:25:46\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0288202478050434\n",
      "2022-01-12 17:25:48\n",
      "F1-score for SVM: 0.0288202478050434\n",
      "\n",
      "XGB's score = 0.08774467671735656\n",
      "2022-01-12 17:25:54\n",
      "F1-score for XGB: 0.08774467671735656\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_internet_CIC.csv\n",
      "2022-01-12 17:28:00\n",
      "Evaluating SCAN_BoT-IoT_CIC's ML models on SCAN_internet_CIC data set\n",
      "\n",
      "DT's score = 0.029005199877649938\n",
      "2022-01-12 17:28:12\n",
      "F1-score for DT: 0.029005199877649938\n",
      "\n",
      "LR's score = 0.02259170284392656\n",
      "2022-01-12 17:28:22\n",
      "F1-score for LR: 0.02259170284392656\n",
      "\n",
      "MLP's score = 0.009358827157777859\n",
      "2022-01-12 17:28:31\n",
      "F1-score for MLP: 0.009358827157777859\n",
      "\n",
      "NB's score = 0.010999095784300101\n",
      "2022-01-12 17:31:59\n",
      "F1-score for NB: 0.010999095784300101\n",
      "\n",
      "SVM's score = 0.02290026885641279\n",
      "2022-01-12 17:32:35\n",
      "F1-score for SVM: 0.02290026885641279\n",
      "\n",
      "XGB's score = 0.02017885044380465\n",
      "2022-01-12 17:33:06\n",
      "F1-score for XGB: 0.02017885044380465\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 4 $TYPE_OF_TEST"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2059ae75-d19c-495d-b192-be92342b27c7",
   "metadata": {},
   "source": [
    "## Test internet AB-TRAP data set models for scanning attack detection on all other data sets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b8e3741e-9241-41a6-b9a1-6010edf83852",
   "metadata": {
    "collapsed": true,
    "jupyter": {
     "outputs_hidden": true
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No Overwrite selected. Skipping data sets already tested\n",
      "Target Class: Scanning\\Reconnaissance selected, exclude other attacks from Benign data\n",
      "loading models from SCAN_internet_CIC\n",
      "Models fetched: ['SCAN_internet_CIC_DT.joblib', 'SCAN_internet_CIC_LR.joblib', 'SCAN_internet_CIC_MLP.joblib', 'SCAN_internet_CIC_NB.joblib', 'SCAN_internet_CIC_SVM.joblib', 'SCAN_internet_CIC_XGB.joblib']\n",
      "DT's index of best performance: 5\n",
      "LR's index of best performance: 0\n",
      "MLP's index of best performance: 0\n",
      "NB's index of best performance: 0\n",
      "SVM's index of best performance: 0\n",
      "XGB's index of best performance: 0\n",
      "DT\n",
      "Found F1-score file for SCAN_internet_CIC data set\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_AB-TRAP_CIC.csv\n",
      "2022-01-10 12:18:19\n",
      "Evaluating SCAN_internet_CIC's ML models on SCAN_AB-TRAP_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-10 12:18:30\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-10 12:18:40\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-10 12:18:49\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 12:24:01\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.0\n",
      "2022-01-10 12:25:16\n",
      "F1-score for SVM: 0.0\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-10 12:26:07\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_NB15_CIC.csv\n",
      "2022-01-10 12:26:21\n",
      "Evaluating SCAN_internet_CIC's ML models on SCAN_NB15_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-10 12:26:21\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-10 12:26:21\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-10 12:26:23\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 12:26:23\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.017670157068062825\n",
      "2022-01-10 12:26:23\n",
      "F1-score for SVM: 0.017670157068062825\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-10 12:26:23\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_CIC-IDS_CIC.csv\n",
      "2022-01-10 12:29:04\n",
      "Evaluating SCAN_internet_CIC's ML models on SCAN_CIC-IDS_CIC data set\n",
      "\n",
      "DT's score = 0.13372269878931678\n",
      "2022-01-10 12:29:06\n",
      "F1-score for DT: 0.13372269878931678\n",
      "\n",
      "LR's score = 0.0\n",
      "2022-01-10 12:29:08\n",
      "F1-score for LR: 0.0\n",
      "\n",
      "MLP's score = 0.04765927424969598\n",
      "2022-01-10 12:29:11\n",
      "F1-score for MLP: 0.04765927424969598\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 12:29:15\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.2606229063508082\n",
      "2022-01-10 12:29:17\n",
      "F1-score for SVM: 0.2606229063508082\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-10 12:29:22\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_ToN-IoT_CIC.csv\n",
      "2022-01-10 12:31:17\n",
      "Evaluating SCAN_internet_CIC's ML models on SCAN_ToN-IoT_CIC data set\n",
      "\n",
      "DT's score = 0.0\n",
      "2022-01-10 12:31:20\n",
      "F1-score for DT: 0.0\n",
      "\n",
      "LR's score = 0.028048214574845465\n",
      "2022-01-10 12:31:22\n",
      "F1-score for LR: 0.028048214574845465\n",
      "\n",
      "MLP's score = 0.0\n",
      "2022-01-10 12:31:25\n",
      "F1-score for MLP: 0.0\n",
      "\n",
      "NB's score = 0.0\n",
      "2022-01-10 12:31:30\n",
      "F1-score for NB: 0.0\n",
      "\n",
      "SVM's score = 0.028353352539971514\n",
      "2022-01-10 12:31:33\n",
      "F1-score for SVM: 0.028353352539971514\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-10 12:31:38\n",
      "F1-score for XGB: 0.0\n",
      "\n",
      "reading file: \n",
      "Loading data set from existing file: SCAN_BoT-IoT_CIC.csv\n",
      "2022-01-10 12:36:09\n",
      "Evaluating SCAN_internet_CIC's ML models on SCAN_BoT-IoT_CIC data set\n",
      "\n",
      "DT's score = 9.107919164940432e-06\n",
      "2022-01-10 12:36:15\n",
      "F1-score for DT: 9.107919164940432e-06\n",
      "\n",
      "LR's score = 0.0188326016299295\n",
      "2022-01-10 12:36:19\n",
      "F1-score for LR: 0.0188326016299295\n",
      "\n",
      "MLP's score = 0.08414906040980996\n",
      "2022-01-10 12:36:26\n",
      "F1-score for MLP: 0.08414906040980996\n",
      "\n",
      "NB's score = 1.7077387319129122e-06\n",
      "2022-01-10 12:36:33\n",
      "F1-score for NB: 1.7077387319129122e-06\n",
      "\n",
      "SVM's score = 0.541543051175439\n",
      "2022-01-10 12:36:37\n",
      "F1-score for SVM: 0.541543051175439\n",
      "\n",
      "XGB's score = 0.0\n",
      "2022-01-10 12:36:45\n",
      "F1-score for XGB: 0.0\n",
      "\n"
     ]
    }
   ],
   "source": [
    "%run supportFiles/$METHOD $MAX_NUM_FILES $CIC_FEATURES 5 $TYPE_OF_TEST"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}