PréProcessing_CSE-CIC-IDS2018.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import MinMaxScaler, StandardScaler\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.svm import OneClassSVM\n",
    "from sklearn.pipeline import Pipeline\n",
    "import os\n",
    "import glob\n",
    "import csv\n",
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "from sklearn import preprocessing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "os.chdir(\"D:/Data\")\n",
    "csv_data_path = 'D:/Data/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a dataframe and the csv file\n",
    "\n",
    "files = ['02-14-2018.csv',\n",
    "        '02-15-2018.csv',\n",
    "        '02-16-2018.csv',\n",
    "        '02-20-2018.csv',\n",
    "        '02-21-2018.csv',\n",
    "        '02-22-2018.csv',\n",
    "        '02-23-2018.csv',\n",
    "        '02-28-2018.csv',\n",
    "        '03-01-2018.csv',\n",
    "        '03-02-2018.csv']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fusionfiles(files):\n",
    "    dfs = []\n",
    "    for i in range (10):\n",
    "        df = pd.read_csv(files[i],nrows=10000)\n",
    "        dfs.append(df)\n",
    "    return dfs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\hp\\anaconda3\\lib\\site-packages\\IPython\\core\\interactiveshell.py:3338: DtypeWarning: Columns (0,1,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59,60,61,62,63,64,65,66,67,68,69,70,71,72,73,74,75,76,77,78) have mixed types.Specify dtype option on import or set low_memory=False.\n",
      "  if (await self.run_code(code, result,  async_=asy)):\n"
     ]
    }
   ],
   "source": [
    "dfs = []\n",
    "dfs = fusionfiles(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 100000 entries, 0 to 9999\n",
      "Data columns (total 79 columns):\n",
      " #   Column             Non-Null Count   Dtype \n",
      "---  ------             --------------   ----- \n",
      " 0   Dst Port           100000 non-null  object\n",
      " 1   Protocol           100000 non-null  object\n",
      " 2   Flow Duration      100000 non-null  object\n",
      " 3   Tot Fwd Pkts       100000 non-null  object\n",
      " 4   Tot Bwd Pkts       100000 non-null  object\n",
      " 5   TotLen Fwd Pkts    100000 non-null  object\n",
      " 6   TotLen Bwd Pkts    100000 non-null  object\n",
      " 7   Fwd Pkt Len Max    100000 non-null  object\n",
      " 8   Fwd Pkt Len Min    100000 non-null  object\n",
      " 9   Fwd Pkt Len Mean   100000 non-null  object\n",
      " 10  Fwd Pkt Len Std    100000 non-null  object\n",
      " 11  Bwd Pkt Len Max    100000 non-null  object\n",
      " 12  Bwd Pkt Len Min    100000 non-null  object\n",
      " 13  Bwd Pkt Len Mean   100000 non-null  object\n",
      " 14  Bwd Pkt Len Std    100000 non-null  object\n",
      " 15  Flow Byts/s        99648 non-null   object\n",
      " 16  Flow Pkts/s        100000 non-null  object\n",
      " 17  Flow IAT Mean      100000 non-null  object\n",
      " 18  Flow IAT Std       100000 non-null  object\n",
      " 19  Flow IAT Max       100000 non-null  object\n",
      " 20  Flow IAT Min       100000 non-null  object\n",
      " 21  Fwd IAT Tot        100000 non-null  object\n",
      " 22  Fwd IAT Mean       100000 non-null  object\n",
      " 23  Fwd IAT Std        100000 non-null  object\n",
      " 24  Fwd IAT Max        100000 non-null  object\n",
      " 25  Fwd IAT Min        100000 non-null  object\n",
      " 26  Bwd IAT Tot        100000 non-null  object\n",
      " 27  Bwd IAT Mean       100000 non-null  object\n",
      " 28  Bwd IAT Std        100000 non-null  object\n",
      " 29  Bwd IAT Max        100000 non-null  object\n",
      " 30  Bwd IAT Min        100000 non-null  object\n",
      " 31  Fwd PSH Flags      100000 non-null  object\n",
      " 32  Bwd PSH Flags      100000 non-null  object\n",
      " 33  Fwd URG Flags      100000 non-null  object\n",
      " 34  Bwd URG Flags      100000 non-null  object\n",
      " 35  Fwd Header Len     100000 non-null  object\n",
      " 36  Bwd Header Len     100000 non-null  object\n",
      " 37  Fwd Pkts/s         100000 non-null  object\n",
      " 38  Bwd Pkts/s         100000 non-null  object\n",
      " 39  Pkt Len Min        100000 non-null  object\n",
      " 40  Pkt Len Max        100000 non-null  object\n",
      " 41  Pkt Len Mean       100000 non-null  object\n",
      " 42  Pkt Len Std        100000 non-null  object\n",
      " 43  Pkt Len Var        100000 non-null  object\n",
      " 44  FIN Flag Cnt       100000 non-null  object\n",
      " 45  SYN Flag Cnt       100000 non-null  object\n",
      " 46  RST Flag Cnt       100000 non-null  object\n",
      " 47  PSH Flag Cnt       100000 non-null  object\n",
      " 48  ACK Flag Cnt       100000 non-null  object\n",
      " 49  URG Flag Cnt       100000 non-null  object\n",
      " 50  CWE Flag Count     100000 non-null  object\n",
      " 51  ECE Flag Cnt       100000 non-null  object\n",
      " 52  Down/Up Ratio      100000 non-null  object\n",
      " 53  Pkt Size Avg       100000 non-null  object\n",
      " 54  Fwd Seg Size Avg   100000 non-null  object\n",
      " 55  Bwd Seg Size Avg   100000 non-null  object\n",
      " 56  Fwd Byts/b Avg     100000 non-null  object\n",
      " 57  Fwd Pkts/b Avg     100000 non-null  object\n",
      " 58  Fwd Blk Rate Avg   100000 non-null  object\n",
      " 59  Bwd Byts/b Avg     100000 non-null  object\n",
      " 60  Bwd Pkts/b Avg     100000 non-null  object\n",
      " 61  Bwd Blk Rate Avg   100000 non-null  object\n",
      " 62  Subflow Fwd Pkts   100000 non-null  object\n",
      " 63  Subflow Fwd Byts   100000 non-null  object\n",
      " 64  Subflow Bwd Pkts   100000 non-null  object\n",
      " 65  Subflow Bwd Byts   100000 non-null  object\n",
      " 66  Init Fwd Win Byts  100000 non-null  object\n",
      " 67  Init Bwd Win Byts  100000 non-null  object\n",
      " 68  Fwd Act Data Pkts  100000 non-null  object\n",
      " 69  Fwd Seg Size Min   100000 non-null  object\n",
      " 70  Active Mean        100000 non-null  object\n",
      " 71  Active Std         100000 non-null  object\n",
      " 72  Active Max         100000 non-null  object\n",
      " 73  Active Min         100000 non-null  object\n",
      " 74  Idle Mean          100000 non-null  object\n",
      " 75  Idle Std           100000 non-null  object\n",
      " 76  Idle Max           100000 non-null  object\n",
      " 77  Idle Min           100000 non-null  object\n",
      " 78  Label              100000 non-null  object\n",
      "dtypes: object(79)\n",
      "memory usage: 30.9+ MB\n"
     ]
    }
   ],
   "source": [
    "# Concatenation de toute les données dans un seul DataFrame\n",
    "data_df = pd.concat(dfs)\n",
    "data_df = data_df.drop([ 'Flow ID', 'Src IP', 'Dst IP', 'Src Port','Timestamp'], axis=1)\n",
    "data_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100000, 79)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Benign                      43083\n",
       "DDoS attacks-LOIC-HTTP       9934\n",
       "DoS attacks-GoldenEye        9931\n",
       "FTP-BruteForce               9904\n",
       "DoS attacks-SlowHTTPTest     9899\n",
       "Bot                          8520\n",
       "DDOS attack-HOIC             6070\n",
       "DDOS attack-LOIC-UDP         1730\n",
       "Brute Force -Web              611\n",
       "Brute Force -XSS              230\n",
       "SQL Injection                  87\n",
       "Label                           1\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data_df['Label'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Dst Port         float64\n",
       "Protocol         float64\n",
       "Flow Duration    float64\n",
       "Tot Fwd Pkts     float64\n",
       "Tot Bwd Pkts     float64\n",
       "                  ...   \n",
       "Idle Mean        float64\n",
       "Idle Std         float64\n",
       "Idle Max         float64\n",
       "Idle Min         float64\n",
       "Label              int32\n",
       "Length: 79, dtype: object"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#suppression des champs infinity et inf\n",
    "data_df = data_df.replace('Infinity', 'inf')\n",
    "#transformation des données en numérique\n",
    "for column in data_df.columns:\n",
    "    if(column!=\"Label\"):\n",
    "        data_df[column] = pd.to_numeric(data_df[column], errors='coerce')\n",
    "data_df.dtypes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "#remplacement des données vide par la moyenne de la variable dont lequel elles sont présente\n",
    "data_df.dropna()\n",
    "data_df.fillna(data_df.mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#création du fichier avec les variables qu'il faut et leur format \n",
    "data_df.to_csv (r'dt_2018_dataframe.csv', index = False, header=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "data_df = pd.read_csv('dt_2018_dataframe.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "#séparation du dataset en données d'entrainement et de test\n",
    "train = data_df.groupby('Label').apply(pd.DataFrame.sample, frac=0.8).reset_index(level='Label', drop=True)\n",
    "test = data_df.drop(train.index)\n",
    "label = train.Label.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    45533\n",
       "0    34466\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Encodez les étiquettes cibles des données d'entrainement  avec une valeur 0 pour les comportement normale et 1 pour les attaques\n",
    "d1 = train.replace('Benign', 0)\n",
    "d2 = d1.replace('Bot', 1)\n",
    "d3 = d2.replace('Brute Force -Web', 1)\n",
    "d4 = d3.replace('Brute Force -XSS', 1)\n",
    "d5 = d4.replace('DDOS attack-HOIC', 1)\n",
    "d6 = d5.replace('DDOS attack-LOIC-UDP', 1)\n",
    "d7 = d6.replace('DDoS attacks-LOIC-HTTP', 1)\n",
    "d8 = d7.replace('DoS attacks-GoldenEye', 1)\n",
    "d9 = d8.replace('DoS attacks-SlowHTTPTest', 1)\n",
    "d10 = d9.replace('FTP-BruteForce', 1)\n",
    "d11 = d10.replace('SQL Injection', 1)\n",
    "d12_label = d11.Label.copy()\n",
    "d12_label.unique()\n",
    "d12_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1    11383\n",
       "0     8617\n",
       "Name: Label, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Encodez les étiquettes cibles des données de testes avec une valeur 0 pour les comportement normale et 1 pour les attaques\n",
    "test_label = test.Label.copy() #ra kanggo\n",
    "test_label.unique() #ra kanggo\n",
    "a1_label = test.Label.copy()\n",
    "a1_label.unique()\n",
    "a1 = test.replace('Benign', 0)\n",
    "a2 = a1.replace('Bot', 1)\n",
    "a3 = a2.replace('Brute Force -Web', 1)\n",
    "a4 = a3.replace('Brute Force -XSS', 1)\n",
    "a5 = a4.replace('DDOS attack-HOIC', 1)\n",
    "a6 = a5.replace('DDOS attack-LOIC-UDP', 1)\n",
    "a7 = a6.replace('DDoS attacks-LOIC-HTTP', 1)\n",
    "a8 = a7.replace('DoS attacks-GoldenEye', 1)\n",
    "a9 = a8.replace('DoS attacks-SlowHTTPTest', 1)\n",
    "a10 = a9.replace('FTP-BruteForce', 1)\n",
    "a11 = a10.replace('SQL Injection', 1)\n",
    "a12_label = a11.Label.copy()\n",
    "a12_label.unique()\n",
    "a12_label.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Length of Categories for Protocol are 3\n",
      "Categories for Protocol are Int64Index([0, 6, 17], dtype='int64') \n",
      "\n"
     ]
    }
   ],
   "source": [
    "#création de 3 nouvelle variable à partir de la variable Protocol, car elle à que 3 données distinctes parmis toutes les observation\n",
    "category_variables = [\"Protocol\"]\n",
    "\n",
    "for cv in category_variables:\n",
    "    d11[cv] = d11[cv].astype(\"category\")\n",
    "    a11[cv] = a11[cv].astype(\"category\")\n",
    "    \n",
    "    print(\"Length of Categories for {} are {}\".format(cv , len(d11[cv].cat.categories)))\n",
    "    print(\"Categories for {} are {} \\n\".format(cv ,d11[cv].cat.categories))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "dummy_variables_2labels = category_variables\n",
    "    \n",
    "class preprocessing:\n",
    "    train_labels = pd.get_dummies(d11, columns = dummy_variables_2labels, prefix=dummy_variables_2labels)\n",
    "    test_labels = pd.get_dummies(a11, columns = dummy_variables_2labels, prefix=dummy_variables_2labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 20000 entries, 0 to 99997\n",
      "Data columns (total 81 columns):\n",
      " #   Column             Non-Null Count  Dtype  \n",
      "---  ------             --------------  -----  \n",
      " 0   Dst Port           20000 non-null  int64  \n",
      " 1   Flow Duration      20000 non-null  int64  \n",
      " 2   Tot Fwd Pkts       20000 non-null  int64  \n",
      " 3   Tot Bwd Pkts       20000 non-null  int64  \n",
      " 4   TotLen Fwd Pkts    20000 non-null  int64  \n",
      " 5   TotLen Bwd Pkts    20000 non-null  int64  \n",
      " 6   Fwd Pkt Len Max    20000 non-null  int64  \n",
      " 7   Fwd Pkt Len Min    20000 non-null  int64  \n",
      " 8   Fwd Pkt Len Mean   20000 non-null  float64\n",
      " 9   Fwd Pkt Len Std    20000 non-null  float64\n",
      " 10  Bwd Pkt Len Max    20000 non-null  int64  \n",
      " 11  Bwd Pkt Len Min    20000 non-null  int64  \n",
      " 12  Bwd Pkt Len Mean   20000 non-null  float64\n",
      " 13  Bwd Pkt Len Std    20000 non-null  float64\n",
      " 14  Flow Byts/s        20000 non-null  float64\n",
      " 15  Flow Pkts/s        20000 non-null  float64\n",
      " 16  Flow IAT Mean      20000 non-null  float64\n",
      " 17  Flow IAT Std       20000 non-null  float64\n",
      " 18  Flow IAT Max       20000 non-null  int64  \n",
      " 19  Flow IAT Min       20000 non-null  int64  \n",
      " 20  Fwd IAT Tot        20000 non-null  int64  \n",
      " 21  Fwd IAT Mean       20000 non-null  float64\n",
      " 22  Fwd IAT Std        20000 non-null  float64\n",
      " 23  Fwd IAT Max        20000 non-null  int64  \n",
      " 24  Fwd IAT Min        20000 non-null  int64  \n",
      " 25  Bwd IAT Tot        20000 non-null  int64  \n",
      " 26  Bwd IAT Mean       20000 non-null  float64\n",
      " 27  Bwd IAT Std        20000 non-null  float64\n",
      " 28  Bwd IAT Max        20000 non-null  int64  \n",
      " 29  Bwd IAT Min        20000 non-null  int64  \n",
      " 30  Fwd PSH Flags      20000 non-null  int64  \n",
      " 31  Bwd PSH Flags      20000 non-null  int64  \n",
      " 32  Fwd URG Flags      20000 non-null  int64  \n",
      " 33  Bwd URG Flags      20000 non-null  int64  \n",
      " 34  Fwd Header Len     20000 non-null  int64  \n",
      " 35  Bwd Header Len     20000 non-null  int64  \n",
      " 36  Fwd Pkts/s         20000 non-null  float64\n",
      " 37  Bwd Pkts/s         20000 non-null  float64\n",
      " 38  Pkt Len Min        20000 non-null  int64  \n",
      " 39  Pkt Len Max        20000 non-null  int64  \n",
      " 40  Pkt Len Mean       20000 non-null  float64\n",
      " 41  Pkt Len Std        20000 non-null  float64\n",
      " 42  Pkt Len Var        20000 non-null  float64\n",
      " 43  FIN Flag Cnt       20000 non-null  int64  \n",
      " 44  SYN Flag Cnt       20000 non-null  int64  \n",
      " 45  RST Flag Cnt       20000 non-null  int64  \n",
      " 46  PSH Flag Cnt       20000 non-null  int64  \n",
      " 47  ACK Flag Cnt       20000 non-null  int64  \n",
      " 48  URG Flag Cnt       20000 non-null  int64  \n",
      " 49  CWE Flag Count     20000 non-null  int64  \n",
      " 50  ECE Flag Cnt       20000 non-null  int64  \n",
      " 51  Down/Up Ratio      20000 non-null  int64  \n",
      " 52  Pkt Size Avg       20000 non-null  float64\n",
      " 53  Fwd Seg Size Avg   20000 non-null  float64\n",
      " 54  Bwd Seg Size Avg   20000 non-null  float64\n",
      " 55  Fwd Byts/b Avg     20000 non-null  int64  \n",
      " 56  Fwd Pkts/b Avg     20000 non-null  int64  \n",
      " 57  Fwd Blk Rate Avg   20000 non-null  int64  \n",
      " 58  Bwd Byts/b Avg     20000 non-null  int64  \n",
      " 59  Bwd Pkts/b Avg     20000 non-null  int64  \n",
      " 60  Bwd Blk Rate Avg   20000 non-null  int64  \n",
      " 61  Subflow Fwd Pkts   20000 non-null  int64  \n",
      " 62  Subflow Fwd Byts   20000 non-null  int64  \n",
      " 63  Subflow Bwd Pkts   20000 non-null  int64  \n",
      " 64  Subflow Bwd Byts   20000 non-null  int64  \n",
      " 65  Init Fwd Win Byts  20000 non-null  int64  \n",
      " 66  Init Bwd Win Byts  20000 non-null  int64  \n",
      " 67  Fwd Act Data Pkts  20000 non-null  int64  \n",
      " 68  Fwd Seg Size Min   20000 non-null  int64  \n",
      " 69  Active Mean        20000 non-null  float64\n",
      " 70  Active Std         20000 non-null  float64\n",
      " 71  Active Max         20000 non-null  int64  \n",
      " 72  Active Min         20000 non-null  int64  \n",
      " 73  Idle Mean          20000 non-null  float64\n",
      " 74  Idle Std           20000 non-null  float64\n",
      " 75  Idle Max           20000 non-null  int64  \n",
      " 76  Idle Min           20000 non-null  int64  \n",
      " 77  Label              20000 non-null  int64  \n",
      " 78  Protocol_0         20000 non-null  uint8  \n",
      " 79  Protocol_6         20000 non-null  uint8  \n",
      " 80  Protocol_17        20000 non-null  uint8  \n",
      "dtypes: float64(24), int64(54), uint8(3)\n",
      "memory usage: 12.1 MB\n"
     ]
    }
   ],
   "source": [
    "preprocessing.test_labels.info(verbose=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "preprocessing.train_labels.to_csv(\"D:/Data/preprocessed_train_DT_2018.csv\")\n",
    "preprocessing.test_labels.to_csv(\"D:/Data/preprocessed_test_DT_2018.csv\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}