{ "cells": [ { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.734837Z", "start_time": "2024-07-05T10:00:39.731830Z" } }, "cell_type": "code", "source": [ "import pandas as pd\n", "import matplotlib.pyplot as plt" ], "id": "5fe41d9df9b1696a", "outputs": [], "execution_count": 42 }, { "metadata": {}, "cell_type": "markdown", "source": "", "id": "bb7781999a6a9d6c" }, { "metadata": {}, "cell_type": "markdown", "source": "", "id": "19715e333704acd5" }, { "cell_type": "code", "id": "initial_id", "metadata": { "collapsed": true, "ExecuteTime": { "end_time": "2024-07-05T10:00:39.760940Z", "start_time": "2024-07-05T10:00:39.756880Z" } }, "source": "data = pd.read_csv(r'C:\\Users\\alepa\\PycharmProjects\\pythonProject\\Optimization\\Project\\Attacks Dataset - MNIST.csv')", "outputs": [], "execution_count": 43 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.796278Z", "start_time": "2024-07-05T10:00:39.792498Z" } }, "cell_type": "code", "source": "data = pd.DataFrame(data)", "id": "892270b24ff68120", "outputs": [], "execution_count": 44 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.811442Z", "start_time": "2024-07-05T10:00:39.797284Z" } }, "cell_type": "code", "source": "data", "id": "c7995cab6c423011", "outputs": [ { "data": { "text/plain": [ " epsilon attack_type norm_type targeted step_rule accuracy \\\n", "0 0.005 fw -1 True lipschitz_mnist 1.00 \n", "1 0.005 fw -1 True decay 1.00 \n", "2 0.005 fw -1 True amjo 1.00 \n", "3 0.005 fw -1 True ls 1.00 \n", "4 0.005 fw -1 False lipschitz_mnist 1.00 \n", ".. ... ... ... ... ... ... \n", "355 0.250 fw_pair 2 True ls 0.95 \n", "356 0.250 fw_pair 2 False lipschitz_mnist 0.88 \n", "357 0.250 fw_pair 2 False decay 0.85 \n", "358 0.250 fw_pair 2 False amjo 0.85 \n", "359 0.250 fw_pair 2 False ls 0.85 \n", "\n", " total_queries \n", "0 1948 \n", "1 1948 \n", "2 1948 \n", "3 1948 \n", "4 1876 \n", ".. ... \n", "355 2002 \n", "356 1928 \n", "357 1859 \n", "358 1856 \n", "359 1855 \n", "\n", "[360 rows x 7 columns]" ], "text/html": [ "<div>\n", "<style scoped>\n", " .dataframe tbody tr th:only-of-type {\n", " vertical-align: middle;\n", " }\n", "\n", " .dataframe tbody tr th {\n", " vertical-align: top;\n", " }\n", "\n", " .dataframe thead th {\n", " text-align: right;\n", " }\n", "</style>\n", "<table border=\"1\" class=\"dataframe\">\n", " <thead>\n", " <tr style=\"text-align: right;\">\n", " <th></th>\n", " <th>epsilon</th>\n", " <th>attack_type</th>\n", " <th>norm_type</th>\n", " <th>targeted</th>\n", " <th>step_rule</th>\n", " <th>accuracy</th>\n", " <th>total_queries</th>\n", " </tr>\n", " </thead>\n", " <tbody>\n", " <tr>\n", " <th>0</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>lipschitz_mnist</td>\n", " <td>1.00</td>\n", " <td>1948</td>\n", " </tr>\n", " <tr>\n", " <th>1</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>decay</td>\n", " <td>1.00</td>\n", " <td>1948</td>\n", " </tr>\n", " <tr>\n", " <th>2</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>amjo</td>\n", " <td>1.00</td>\n", " <td>1948</td>\n", " </tr>\n", " <tr>\n", " <th>3</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>ls</td>\n", " <td>1.00</td>\n", " <td>1948</td>\n", " </tr>\n", " <tr>\n", " <th>4</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>False</td>\n", " <td>lipschitz_mnist</td>\n", " <td>1.00</td>\n", " <td>1876</td>\n", " </tr>\n", " <tr>\n", " <th>...</th>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " </tr>\n", " <tr>\n", " <th>355</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>True</td>\n", " <td>ls</td>\n", " <td>0.95</td>\n", " <td>2002</td>\n", " </tr>\n", " <tr>\n", " <th>356</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>lipschitz_mnist</td>\n", " <td>0.88</td>\n", " <td>1928</td>\n", " </tr>\n", " <tr>\n", " <th>357</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>decay</td>\n", " <td>0.85</td>\n", " <td>1859</td>\n", " </tr>\n", " <tr>\n", " <th>358</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>amjo</td>\n", " <td>0.85</td>\n", " <td>1856</td>\n", " </tr>\n", " <tr>\n", " <th>359</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>ls</td>\n", " <td>0.85</td>\n", " <td>1855</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "<p>360 rows × 7 columns</p>\n", "</div>" ] }, "execution_count": 45, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 45 }, { "metadata": {}, "cell_type": "markdown", "source": "", "id": "451fe2c1b077182c" }, { "metadata": {}, "cell_type": "markdown", "source": "", "id": "33a113363bf0a03e" }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.829743Z", "start_time": "2024-07-05T10:00:39.811442Z" } }, "cell_type": "code", "source": "data.describe()", "id": "c5d432a3e81f99e5", "outputs": [ { "data": { "text/plain": [ " epsilon norm_type accuracy total_queries\n", "count 360.000000 360.000000 360.000000 360.000000\n", "mean 0.083000 0.666667 0.620306 1370.322222\n", "std 0.090325 1.248955 0.426050 831.701901\n", "min 0.005000 -1.000000 0.010000 124.000000\n", "25% 0.010000 -1.000000 0.090000 202.000000\n", "50% 0.050000 1.000000 0.930000 1984.000000\n", "75% 0.100000 2.000000 1.000000 2002.000000\n", "max 0.250000 2.000000 1.000000 2002.000000" ], "text/html": [ "<div>\n", "<style scoped>\n", " .dataframe tbody tr th:only-of-type {\n", " vertical-align: middle;\n", " }\n", "\n", " .dataframe tbody tr th {\n", " vertical-align: top;\n", " }\n", "\n", " .dataframe thead th {\n", " text-align: right;\n", " }\n", "</style>\n", "<table border=\"1\" class=\"dataframe\">\n", " <thead>\n", " <tr style=\"text-align: right;\">\n", " <th></th>\n", " <th>epsilon</th>\n", " <th>norm_type</th>\n", " <th>accuracy</th>\n", " <th>total_queries</th>\n", " </tr>\n", " </thead>\n", " <tbody>\n", " <tr>\n", " <th>count</th>\n", " <td>360.000000</td>\n", " <td>360.000000</td>\n", " <td>360.000000</td>\n", " <td>360.000000</td>\n", " </tr>\n", " <tr>\n", " <th>mean</th>\n", " <td>0.083000</td>\n", " <td>0.666667</td>\n", " <td>0.620306</td>\n", " <td>1370.322222</td>\n", " </tr>\n", " <tr>\n", " <th>std</th>\n", " <td>0.090325</td>\n", " <td>1.248955</td>\n", " <td>0.426050</td>\n", " <td>831.701901</td>\n", " </tr>\n", " <tr>\n", " <th>min</th>\n", " <td>0.005000</td>\n", " <td>-1.000000</td>\n", " <td>0.010000</td>\n", " <td>124.000000</td>\n", " </tr>\n", " <tr>\n", " <th>25%</th>\n", " <td>0.010000</td>\n", " <td>-1.000000</td>\n", " <td>0.090000</td>\n", " <td>202.000000</td>\n", " </tr>\n", " <tr>\n", " <th>50%</th>\n", " <td>0.050000</td>\n", " <td>1.000000</td>\n", " <td>0.930000</td>\n", " <td>1984.000000</td>\n", " </tr>\n", " <tr>\n", " <th>75%</th>\n", " <td>0.100000</td>\n", " <td>2.000000</td>\n", " <td>1.000000</td>\n", " <td>2002.000000</td>\n", " </tr>\n", " <tr>\n", " <th>max</th>\n", " <td>0.250000</td>\n", " <td>2.000000</td>\n", " <td>1.000000</td>\n", " <td>2002.000000</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "</div>" ] }, "execution_count": 46, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 46 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.835420Z", "start_time": "2024-07-05T10:00:39.830750Z" } }, "cell_type": "code", "source": "print(data.groupby('attack_type')[['accuracy', 'total_queries']].mean())", "id": "eb4d75342aef24cb", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " accuracy total_queries\n", "attack_type \n", "fw 0.620250 1052.333333\n", "fw_away 0.621083 2002.000000\n", "fw_pair 0.619583 1056.633333\n" ] } ], "execution_count": 47 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.845144Z", "start_time": "2024-07-05T10:00:39.835420Z" } }, "cell_type": "code", "source": "print(data.groupby('epsilon')['accuracy'].mean())", "id": "98b0dc78de43c13", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "epsilon\n", "0.005 0.702917\n", "0.010 0.701250\n", "0.050 0.690278\n", "0.100 0.634028\n", "0.250 0.373056\n", "Name: accuracy, dtype: float64\n" ] } ], "execution_count": 48 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.855479Z", "start_time": "2024-07-05T10:00:39.845144Z" } }, "cell_type": "code", "source": "print(data.groupby('norm_type')['accuracy'].mean())", "id": "3ff32ea5ba11215e", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "norm_type\n", "-1 0.774917\n", " 1 0.109167\n", " 2 0.976833\n", "Name: accuracy, dtype: float64\n" ] } ], "execution_count": 49 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.866539Z", "start_time": "2024-07-05T10:00:39.856486Z" } }, "cell_type": "code", "source": "print(data.groupby('targeted')['accuracy'].mean())", "id": "706907346e0e9426", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "targeted\n", "False 0.597000\n", "True 0.643611\n", "Name: accuracy, dtype: float64\n" ] } ], "execution_count": 50 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.878124Z", "start_time": "2024-07-05T10:00:39.866539Z" } }, "cell_type": "code", "source": "print(data.groupby('step_rule')[['accuracy', 'total_queries']].mean())", "id": "608a884c74003aee", "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ " accuracy total_queries\n", "step_rule \n", "amjo 0.609111 1365.366667\n", "decay 0.606778 1365.177778\n", "lipschitz_mnist 0.630333 1383.777778\n", "ls 0.635000 1366.966667\n" ] } ], "execution_count": 51 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:00:39.896723Z", "start_time": "2024-07-05T10:00:39.879132Z" } }, "cell_type": "code", "source": [ "threshold = 0.1\n", "nice_data = pd.DataFrame(data[data['accuracy'] < threshold])\n", "nice_data" ], "id": "ecd84843f50700e3", "outputs": [ { "data": { "text/plain": [ " epsilon attack_type norm_type targeted step_rule accuracy \\\n", "8 0.005 fw 1 True lipschitz_mnist 0.09 \n", "9 0.005 fw 1 True decay 0.09 \n", "10 0.005 fw 1 True amjo 0.09 \n", "12 0.005 fw 1 False lipschitz_mnist 0.09 \n", "13 0.005 fw 1 False decay 0.09 \n", ".. ... ... ... ... ... ... \n", "346 0.250 fw_pair 1 True amjo 0.09 \n", "348 0.250 fw_pair 1 False lipschitz_mnist 0.09 \n", "349 0.250 fw_pair 1 False decay 0.09 \n", "350 0.250 fw_pair 1 False amjo 0.09 \n", "351 0.250 fw_pair 1 False ls 0.09 \n", "\n", " total_queries \n", "8 202 \n", "9 202 \n", "10 202 \n", "12 202 \n", "13 202 \n", ".. ... \n", "346 202 \n", "348 202 \n", "349 202 \n", "350 202 \n", "351 202 \n", "\n", "[126 rows x 7 columns]" ], "text/html": [ "<div>\n", "<style scoped>\n", " .dataframe tbody tr th:only-of-type {\n", " vertical-align: middle;\n", " }\n", "\n", " .dataframe tbody tr th {\n", " vertical-align: top;\n", " }\n", "\n", " .dataframe thead th {\n", " text-align: right;\n", " }\n", "</style>\n", "<table border=\"1\" class=\"dataframe\">\n", " <thead>\n", " <tr style=\"text-align: right;\">\n", " <th></th>\n", " <th>epsilon</th>\n", " <th>attack_type</th>\n", " <th>norm_type</th>\n", " <th>targeted</th>\n", " <th>step_rule</th>\n", " <th>accuracy</th>\n", " <th>total_queries</th>\n", " </tr>\n", " </thead>\n", " <tbody>\n", " <tr>\n", " <th>8</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>1</td>\n", " <td>True</td>\n", " <td>lipschitz_mnist</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>9</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>1</td>\n", " <td>True</td>\n", " <td>decay</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>10</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>1</td>\n", " <td>True</td>\n", " <td>amjo</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>12</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>1</td>\n", " <td>False</td>\n", " <td>lipschitz_mnist</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>13</th>\n", " <td>0.005</td>\n", " <td>fw</td>\n", " <td>1</td>\n", " <td>False</td>\n", " <td>decay</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>...</th>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " </tr>\n", " <tr>\n", " <th>346</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>1</td>\n", " <td>True</td>\n", " <td>amjo</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>348</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>1</td>\n", " <td>False</td>\n", " <td>lipschitz_mnist</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>349</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>1</td>\n", " <td>False</td>\n", " <td>decay</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>350</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>1</td>\n", " <td>False</td>\n", " <td>amjo</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " <tr>\n", " <th>351</th>\n", " <td>0.250</td>\n", " <td>fw_pair</td>\n", " <td>1</td>\n", " <td>False</td>\n", " <td>ls</td>\n", " <td>0.09</td>\n", " <td>202</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "<p>126 rows × 7 columns</p>\n", "</div>" ] }, "execution_count": 52, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 52 }, { "metadata": { "ExecuteTime": { "end_time": "2024-07-05T10:01:01.526552Z", "start_time": "2024-07-05T10:01:01.516631Z" } }, "cell_type": "code", "source": [ "nice_data = pd.DataFrame(data[data['epsilon'] == 0.25])\n", "nice_data" ], "id": "457c7a104b6e2217", "outputs": [ { "data": { "text/plain": [ " epsilon attack_type norm_type targeted step_rule accuracy \\\n", "288 0.25 fw -1 True lipschitz_mnist 0.51 \n", "289 0.25 fw -1 True decay 0.05 \n", "290 0.25 fw -1 True amjo 0.04 \n", "291 0.25 fw -1 True ls 0.04 \n", "292 0.25 fw -1 False lipschitz_mnist 0.05 \n", ".. ... ... ... ... ... ... \n", "355 0.25 fw_pair 2 True ls 0.95 \n", "356 0.25 fw_pair 2 False lipschitz_mnist 0.88 \n", "357 0.25 fw_pair 2 False decay 0.85 \n", "358 0.25 fw_pair 2 False amjo 0.85 \n", "359 0.25 fw_pair 2 False ls 0.85 \n", "\n", " total_queries \n", "288 1458 \n", "289 871 \n", "290 871 \n", "291 851 \n", "292 558 \n", ".. ... \n", "355 2002 \n", "356 1928 \n", "357 1859 \n", "358 1856 \n", "359 1855 \n", "\n", "[72 rows x 7 columns]" ], "text/html": [ "<div>\n", "<style scoped>\n", " .dataframe tbody tr th:only-of-type {\n", " vertical-align: middle;\n", " }\n", "\n", " .dataframe tbody tr th {\n", " vertical-align: top;\n", " }\n", "\n", " .dataframe thead th {\n", " text-align: right;\n", " }\n", "</style>\n", "<table border=\"1\" class=\"dataframe\">\n", " <thead>\n", " <tr style=\"text-align: right;\">\n", " <th></th>\n", " <th>epsilon</th>\n", " <th>attack_type</th>\n", " <th>norm_type</th>\n", " <th>targeted</th>\n", " <th>step_rule</th>\n", " <th>accuracy</th>\n", " <th>total_queries</th>\n", " </tr>\n", " </thead>\n", " <tbody>\n", " <tr>\n", " <th>288</th>\n", " <td>0.25</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>lipschitz_mnist</td>\n", " <td>0.51</td>\n", " <td>1458</td>\n", " </tr>\n", " <tr>\n", " <th>289</th>\n", " <td>0.25</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>decay</td>\n", " <td>0.05</td>\n", " <td>871</td>\n", " </tr>\n", " <tr>\n", " <th>290</th>\n", " <td>0.25</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>amjo</td>\n", " <td>0.04</td>\n", " <td>871</td>\n", " </tr>\n", " <tr>\n", " <th>291</th>\n", " <td>0.25</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>True</td>\n", " <td>ls</td>\n", " <td>0.04</td>\n", " <td>851</td>\n", " </tr>\n", " <tr>\n", " <th>292</th>\n", " <td>0.25</td>\n", " <td>fw</td>\n", " <td>-1</td>\n", " <td>False</td>\n", " <td>lipschitz_mnist</td>\n", " <td>0.05</td>\n", " <td>558</td>\n", " </tr>\n", " <tr>\n", " <th>...</th>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " <td>...</td>\n", " </tr>\n", " <tr>\n", " <th>355</th>\n", " <td>0.25</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>True</td>\n", " <td>ls</td>\n", " <td>0.95</td>\n", " <td>2002</td>\n", " </tr>\n", " <tr>\n", " <th>356</th>\n", " <td>0.25</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>lipschitz_mnist</td>\n", " <td>0.88</td>\n", " <td>1928</td>\n", " </tr>\n", " <tr>\n", " <th>357</th>\n", " <td>0.25</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>decay</td>\n", " <td>0.85</td>\n", " <td>1859</td>\n", " </tr>\n", " <tr>\n", " <th>358</th>\n", " <td>0.25</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>amjo</td>\n", " <td>0.85</td>\n", " <td>1856</td>\n", " </tr>\n", " <tr>\n", " <th>359</th>\n", " <td>0.25</td>\n", " <td>fw_pair</td>\n", " <td>2</td>\n", " <td>False</td>\n", " <td>ls</td>\n", " <td>0.85</td>\n", " <td>1855</td>\n", " </tr>\n", " </tbody>\n", "</table>\n", "<p>72 rows × 7 columns</p>\n", "</div>" ] }, "execution_count": 53, "metadata": {}, "output_type": "execute_result" } ], "execution_count": 53 } ], "metadata": { "kernelspec": { "display_name": "Python 3", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 2 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython2", "version": "2.7.6" } }, "nbformat": 4, "nbformat_minor": 5 }