4_cnn_adversarial.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# A gentle practical intro to adversarial ML\n",
    "\n",
    "In this tutorial we will train a convolutional neural network and we will show how we can fool the network perturbing an image."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## AML with images\n",
    "\n",
    "First, we will load all the necessary libraries"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "\n",
    "from torchvision import datasets, transforms\n",
    "\n",
    "torch.manual_seed(1234)\n",
    "np.random.seed(1234)\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Our model will be a simple convolutional network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class SimpleCNN(nn.Module):\n",
    "    \"\"\" Basic MNIST model\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self):\n",
    "        super(SimpleCNN, self).__init__()\n",
    "        \n",
    "        self.conv1 = nn.Conv2d(1, 32, 5, padding=2)\n",
    "        self.conv2 = nn.Conv2d(32, 64, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(64 * 7 * 7, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 10)\n",
    "\n",
    "    def forward(self, x):\n",
    "        x = F.max_pool2d(F.relu(self.conv1(x)), 2)\n",
    "        x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
    "        x = x.view(-1, 64 * 7 * 7)  # reshape Variable\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.fc2(x)\n",
    "        return F.log_softmax(x, dim=-1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We may explore the architecture"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "SimpleCNN(\n",
       "  (conv1): Conv2d(1, 32, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
       "  (conv2): Conv2d(32, 64, kernel_size=(5, 5), stride=(1, 1), padding=(2, 2))\n",
       "  (fc1): Linear(in_features=3136, out_features=1024, bias=True)\n",
       "  (fc2): Linear(in_features=1024, out_features=10, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cnn = SimpleCNN()\n",
    "cnn"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "With the help of https://github.com/szagoruyko/pytorchviz we will visualize the network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "from torchviz import make_dot"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/svg+xml": [
       "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\"?>\n",
       "<!DOCTYPE svg PUBLIC \"-//W3C//DTD SVG 1.1//EN\"\n",
       " \"http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd\">\n",
       "<!-- Generated by graphviz version 2.36.0 (20140111.2315)\n",
       " -->\n",
       "<!-- Title: %3 Pages: 1 -->\n",
       "<svg width=\"437pt\" height=\"740pt\"\n",
       " viewBox=\"0.00 0.00 437.00 740.00\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\">\n",
       "<g id=\"graph0\" class=\"graph\" transform=\"scale(1 1) rotate(0) translate(4 736)\">\n",
       "<title>%3</title>\n",
       "<polygon fill=\"white\" stroke=\"none\" points=\"-4,4 -4,-736 433,-736 433,4 -4,4\"/>\n",
       "<!-- 139766289659272 -->\n",
       "<g id=\"node1\" class=\"node\"><title>139766289659272</title>\n",
       "<polygon fill=\"#caff70\" stroke=\"black\" points=\"314.25,-21.5 189.75,-21.5 189.75,-0.5 314.25,-0.5 314.25,-21.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"252\" y=\"-7.9\" font-family=\"Times,serif\" font-size=\"12.00\">LogSoftmaxBackward</text>\n",
       "</g>\n",
       "<!-- 139766289659328 -->\n",
       "<g id=\"node2\" class=\"node\"><title>139766289659328</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"304,-79.5 200,-79.5 200,-58.5 304,-58.5 304,-79.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"252\" y=\"-65.9\" font-family=\"Times,serif\" font-size=\"12.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 139766289659328&#45;&gt;139766289659272 -->\n",
       "<g id=\"edge1\" class=\"edge\"><title>139766289659328&#45;&gt;139766289659272</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M252,-58.2455C252,-50.9382 252,-40.6944 252,-31.7046\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"255.5,-31.6426 252,-21.6427 248.5,-31.6427 255.5,-31.6426\"/>\n",
       "</g>\n",
       "<!-- 139766289659440 -->\n",
       "<g id=\"node3\" class=\"node\"><title>139766289659440</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"187,-150 133,-150 133,-116 187,-116 187,-150\"/>\n",
       "<text text-anchor=\"middle\" x=\"160\" y=\"-136.4\" font-family=\"Times,serif\" font-size=\"12.00\">fc2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"160\" y=\"-123.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (10)</text>\n",
       "</g>\n",
       "<!-- 139766289659440&#45;&gt;139766289659328 -->\n",
       "<g id=\"edge2\" class=\"edge\"><title>139766289659440&#45;&gt;139766289659328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M184.172,-115.71C198.147,-106.292 215.621,-94.5162 229.333,-85.2755\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"231.389,-88.1105 237.726,-79.6195 227.477,-82.3056 231.389,-88.1105\"/>\n",
       "</g>\n",
       "<!-- 139766289659496 -->\n",
       "<g id=\"node4\" class=\"node\"><title>139766289659496</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"299,-143.5 205,-143.5 205,-122.5 299,-122.5 299,-143.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"252\" y=\"-129.9\" font-family=\"Times,serif\" font-size=\"12.00\">ReluBackward0</text>\n",
       "</g>\n",
       "<!-- 139766289659496&#45;&gt;139766289659328 -->\n",
       "<g id=\"edge3\" class=\"edge\"><title>139766289659496&#45;&gt;139766289659328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M252,-122.312C252,-113.718 252,-100.888 252,-90.0855\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"255.5,-89.8241 252,-79.8241 248.5,-89.8241 255.5,-89.8241\"/>\n",
       "</g>\n",
       "<!-- 139766289659832 -->\n",
       "<g id=\"node5\" class=\"node\"><title>139766289659832</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"303,-213.5 199,-213.5 199,-192.5 303,-192.5 303,-213.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"251\" y=\"-199.9\" font-family=\"Times,serif\" font-size=\"12.00\">AddmmBackward</text>\n",
       "</g>\n",
       "<!-- 139766289659832&#45;&gt;139766289659496 -->\n",
       "<g id=\"edge4\" class=\"edge\"><title>139766289659832&#45;&gt;139766289659496</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M251.143,-192.257C251.29,-182.259 251.523,-166.422 251.709,-153.782\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"255.209,-153.775 251.857,-143.725 248.21,-153.672 255.209,-153.775\"/>\n",
       "</g>\n",
       "<!-- 139766289659944 -->\n",
       "<g id=\"node6\" class=\"node\"><title>139766289659944</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"187,-290 133,-290 133,-256 187,-256 187,-290\"/>\n",
       "<text text-anchor=\"middle\" x=\"160\" y=\"-276.4\" font-family=\"Times,serif\" font-size=\"12.00\">fc1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"160\" y=\"-263.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (1024)</text>\n",
       "</g>\n",
       "<!-- 139766289659944&#45;&gt;139766289659832 -->\n",
       "<g id=\"edge5\" class=\"edge\"><title>139766289659944&#45;&gt;139766289659832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M181.566,-255.885C196.103,-245.022 215.197,-230.754 229.682,-219.93\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"232.108,-222.486 238.024,-213.696 227.918,-216.879 232.108,-222.486\"/>\n",
       "</g>\n",
       "<!-- 139766289660000 -->\n",
       "<g id=\"node7\" class=\"node\"><title>139766289660000</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"297,-283.5 205,-283.5 205,-262.5 297,-262.5 297,-283.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"251\" y=\"-269.9\" font-family=\"Times,serif\" font-size=\"12.00\">ViewBackward</text>\n",
       "</g>\n",
       "<!-- 139766289660000&#45;&gt;139766289659832 -->\n",
       "<g id=\"edge6\" class=\"edge\"><title>139766289660000&#45;&gt;139766289659832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M251,-262.257C251,-252.259 251,-236.422 251,-223.782\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"254.5,-223.725 251,-213.725 247.5,-223.725 254.5,-223.725\"/>\n",
       "</g>\n",
       "<!-- 139766289659664 -->\n",
       "<g id=\"node8\" class=\"node\"><title>139766289659664</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"329.25,-353.5 148.75,-353.5 148.75,-332.5 329.25,-332.5 329.25,-353.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"239\" y=\"-339.9\" font-family=\"Times,serif\" font-size=\"12.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 139766289659664&#45;&gt;139766289660000 -->\n",
       "<g id=\"edge7\" class=\"edge\"><title>139766289659664&#45;&gt;139766289660000</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M240.719,-332.257C242.484,-322.259 245.278,-306.422 247.509,-293.782\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"250.993,-294.181 249.284,-283.725 244.099,-292.964 250.993,-294.181\"/>\n",
       "</g>\n",
       "<!-- 139766289660280 -->\n",
       "<g id=\"node9\" class=\"node\"><title>139766289660280</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"286,-417.5 192,-417.5 192,-396.5 286,-396.5 286,-417.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"239\" y=\"-403.9\" font-family=\"Times,serif\" font-size=\"12.00\">ReluBackward0</text>\n",
       "</g>\n",
       "<!-- 139766289660280&#45;&gt;139766289659664 -->\n",
       "<g id=\"edge8\" class=\"edge\"><title>139766289660280&#45;&gt;139766289659664</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M239,-396.312C239,-387.718 239,-374.888 239,-364.085\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"242.5,-363.824 239,-353.824 235.5,-363.824 242.5,-363.824\"/>\n",
       "</g>\n",
       "<!-- 139766289660392 -->\n",
       "<g id=\"node10\" class=\"node\"><title>139766289660392</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"320,-475.5 158,-475.5 158,-454.5 320,-454.5 320,-475.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"239\" y=\"-461.9\" font-family=\"Times,serif\" font-size=\"12.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 139766289660392&#45;&gt;139766289660280 -->\n",
       "<g id=\"edge9\" class=\"edge\"><title>139766289660392&#45;&gt;139766289660280</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M239,-454.246C239,-446.938 239,-436.694 239,-427.705\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"242.5,-427.643 239,-417.643 235.5,-427.643 242.5,-427.643\"/>\n",
       "</g>\n",
       "<!-- 139766289660504 -->\n",
       "<g id=\"node11\" class=\"node\"><title>139766289660504</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"180.25,-539.5 -0.25,-539.5 -0.25,-518.5 180.25,-518.5 180.25,-539.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"90\" y=\"-525.9\" font-family=\"Times,serif\" font-size=\"12.00\">MaxPool2DWithIndicesBackward</text>\n",
       "</g>\n",
       "<!-- 139766289660504&#45;&gt;139766289660392 -->\n",
       "<g id=\"edge10\" class=\"edge\"><title>139766289660504&#45;&gt;139766289660392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M112.955,-518.448C137.928,-508.057 178.059,-491.358 206.318,-479.599\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"207.886,-482.738 215.774,-475.665 205.196,-476.275 207.886,-482.738\"/>\n",
       "</g>\n",
       "<!-- 139766289660728 -->\n",
       "<g id=\"node12\" class=\"node\"><title>139766289660728</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"137,-603.5 43,-603.5 43,-582.5 137,-582.5 137,-603.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"90\" y=\"-589.9\" font-family=\"Times,serif\" font-size=\"12.00\">ReluBackward0</text>\n",
       "</g>\n",
       "<!-- 139766289660728&#45;&gt;139766289660504 -->\n",
       "<g id=\"edge11\" class=\"edge\"><title>139766289660728&#45;&gt;139766289660504</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M90,-582.312C90,-573.718 90,-560.888 90,-550.085\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.5001,-549.824 90,-539.824 86.5001,-549.824 93.5001,-549.824\"/>\n",
       "</g>\n",
       "<!-- 139766289660840 -->\n",
       "<g id=\"node13\" class=\"node\"><title>139766289660840</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"171,-661.5 9,-661.5 9,-640.5 171,-640.5 171,-661.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"90\" y=\"-647.9\" font-family=\"Times,serif\" font-size=\"12.00\">MkldnnConvolutionBackward</text>\n",
       "</g>\n",
       "<!-- 139766289660840&#45;&gt;139766289660728 -->\n",
       "<g id=\"edge12\" class=\"edge\"><title>139766289660840&#45;&gt;139766289660728</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M90,-640.246C90,-632.938 90,-622.694 90,-613.705\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"93.5001,-613.643 90,-603.643 86.5001,-613.643 93.5001,-613.643\"/>\n",
       "</g>\n",
       "<!-- 139766289660952 -->\n",
       "<g id=\"node14\" class=\"node\"><title>139766289660952</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"84.25,-732 3.75,-732 3.75,-698 84.25,-698 84.25,-732\"/>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-718.4\" font-family=\"Times,serif\" font-size=\"12.00\">conv1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"44\" y=\"-705.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (32, 1, 5, 5)</text>\n",
       "</g>\n",
       "<!-- 139766289660952&#45;&gt;139766289660840 -->\n",
       "<g id=\"edge13\" class=\"edge\"><title>139766289660952&#45;&gt;139766289660840</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M56.086,-697.71C62.4746,-689.1 70.3257,-678.518 76.8644,-669.704\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"79.7153,-671.736 82.863,-661.619 74.0936,-667.565 79.7153,-671.736\"/>\n",
       "</g>\n",
       "<!-- 139766289661008 -->\n",
       "<g id=\"node15\" class=\"node\"><title>139766289661008</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"171,-732 103,-732 103,-698 171,-698 171,-732\"/>\n",
       "<text text-anchor=\"middle\" x=\"137\" y=\"-718.4\" font-family=\"Times,serif\" font-size=\"12.00\">conv1.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"137\" y=\"-705.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (32)</text>\n",
       "</g>\n",
       "<!-- 139766289661008&#45;&gt;139766289660840 -->\n",
       "<g id=\"edge14\" class=\"edge\"><title>139766289661008&#45;&gt;139766289660840</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M124.651,-697.71C118.124,-689.1 110.102,-678.518 103.421,-669.704\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"106.122,-667.474 97.2922,-661.619 100.544,-671.703 106.122,-667.474\"/>\n",
       "</g>\n",
       "<!-- 139766289660560 -->\n",
       "<g id=\"node16\" class=\"node\"><title>139766289660560</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"279.25,-546 198.75,-546 198.75,-512 279.25,-512 279.25,-546\"/>\n",
       "<text text-anchor=\"middle\" x=\"239\" y=\"-532.4\" font-family=\"Times,serif\" font-size=\"12.00\">conv2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"239\" y=\"-519.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (64, 32, 5, 5)</text>\n",
       "</g>\n",
       "<!-- 139766289660560&#45;&gt;139766289660392 -->\n",
       "<g id=\"edge15\" class=\"edge\"><title>139766289660560&#45;&gt;139766289660392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M239,-511.71C239,-503.727 239,-494.05 239,-485.659\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"242.5,-485.619 239,-475.619 235.5,-485.62 242.5,-485.619\"/>\n",
       "</g>\n",
       "<!-- 139766289660616 -->\n",
       "<g id=\"node17\" class=\"node\"><title>139766289660616</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"366,-546 298,-546 298,-512 366,-512 366,-546\"/>\n",
       "<text text-anchor=\"middle\" x=\"332\" y=\"-532.4\" font-family=\"Times,serif\" font-size=\"12.00\">conv2.bias</text>\n",
       "<text text-anchor=\"middle\" x=\"332\" y=\"-519.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (64)</text>\n",
       "</g>\n",
       "<!-- 139766289660616&#45;&gt;139766289660392 -->\n",
       "<g id=\"edge16\" class=\"edge\"><title>139766289660616&#45;&gt;139766289660392</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M307.565,-511.71C293.438,-502.292 275.774,-490.516 261.913,-481.276\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"263.691,-478.254 253.429,-475.619 259.808,-484.079 263.691,-478.254\"/>\n",
       "</g>\n",
       "<!-- 139766289660056 -->\n",
       "<g id=\"node18\" class=\"node\"><title>139766289660056</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"412.25,-283.5 339.75,-283.5 339.75,-262.5 412.25,-262.5 412.25,-283.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"376\" y=\"-269.9\" font-family=\"Times,serif\" font-size=\"12.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 139766289660056&#45;&gt;139766289659832 -->\n",
       "<g id=\"edge17\" class=\"edge\"><title>139766289660056&#45;&gt;139766289659832</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M358.356,-262.401C337.324,-250.96 301.97,-231.728 277.614,-218.478\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"279.011,-215.254 268.555,-213.55 275.666,-221.403 279.011,-215.254\"/>\n",
       "</g>\n",
       "<!-- 139766289660224 -->\n",
       "<g id=\"node19\" class=\"node\"><title>139766289660224</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"428.25,-360 347.75,-360 347.75,-326 428.25,-326 428.25,-360\"/>\n",
       "<text text-anchor=\"middle\" x=\"388\" y=\"-346.4\" font-family=\"Times,serif\" font-size=\"12.00\">fc1.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"388\" y=\"-333.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (1024, 3136)</text>\n",
       "</g>\n",
       "<!-- 139766289660224&#45;&gt;139766289660056 -->\n",
       "<g id=\"edge18\" class=\"edge\"><title>139766289660224&#45;&gt;139766289660056</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M385.156,-325.885C383.466,-316.309 381.31,-304.088 379.514,-293.912\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"382.931,-293.134 377.746,-283.895 376.037,-294.351 382.931,-293.134\"/>\n",
       "</g>\n",
       "<!-- 139766289659552 -->\n",
       "<g id=\"node20\" class=\"node\"><title>139766289659552</title>\n",
       "<polygon fill=\"lightgrey\" stroke=\"black\" points=\"391.25,-143.5 318.75,-143.5 318.75,-122.5 391.25,-122.5 391.25,-143.5\"/>\n",
       "<text text-anchor=\"middle\" x=\"355\" y=\"-129.9\" font-family=\"Times,serif\" font-size=\"12.00\">TBackward</text>\n",
       "</g>\n",
       "<!-- 139766289659552&#45;&gt;139766289659328 -->\n",
       "<g id=\"edge19\" class=\"edge\"><title>139766289659552&#45;&gt;139766289659328</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M338.906,-122.312C322.202,-112.257 295.864,-96.4037 276.558,-84.7825\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"278.353,-81.778 267.981,-79.6195 274.743,-87.7754 278.353,-81.778\"/>\n",
       "</g>\n",
       "<!-- 139766289659888 -->\n",
       "<g id=\"node21\" class=\"node\"><title>139766289659888</title>\n",
       "<polygon fill=\"lightblue\" stroke=\"black\" points=\"390.25,-220 321.75,-220 321.75,-186 390.25,-186 390.25,-220\"/>\n",
       "<text text-anchor=\"middle\" x=\"356\" y=\"-206.4\" font-family=\"Times,serif\" font-size=\"12.00\">fc2.weight</text>\n",
       "<text text-anchor=\"middle\" x=\"356\" y=\"-193.4\" font-family=\"Times,serif\" font-size=\"12.00\"> (10, 1024)</text>\n",
       "</g>\n",
       "<!-- 139766289659888&#45;&gt;139766289659552 -->\n",
       "<g id=\"edge20\" class=\"edge\"><title>139766289659888&#45;&gt;139766289659552</title>\n",
       "<path fill=\"none\" stroke=\"black\" d=\"M355.763,-185.885C355.622,-176.309 355.442,-164.088 355.293,-153.912\"/>\n",
       "<polygon fill=\"black\" stroke=\"black\" points=\"358.792,-153.842 355.146,-143.895 351.793,-153.945 358.792,-153.842\"/>\n",
       "</g>\n",
       "</g>\n",
       "</svg>\n"
      ],
      "text/plain": [
       "<graphviz.dot.Digraph at 0x7f1de00bf550>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "random_input = (torch.randn([1, 1, 28, 28]))\n",
    "\n",
    "make_dot(cnn(random_input), params=dict(cnn.named_parameters()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we define the hyperparameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "n_epochs = 6\n",
    "batch_size = 128\n",
    "lr = .001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz to data/MNIST/raw/train-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████▉| 9895936/9912422 [00:20<00:00, 465606.19it/s] "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/train-images-idx3-ubyte.gz to data/MNIST/raw\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "0it [00:00, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz to data/MNIST/raw/train-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/28881 [00:00<?, ?it/s]\u001b[A\n",
      " 57%|█████▋    | 16384/28881 [00:00<00:00, 132447.54it/s]\u001b[A\n",
      "32768it [00:00, 64353.78it/s]                            \u001b[A\n",
      "0it [00:00, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/train-labels-idx1-ubyte.gz to data/MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz to data/MNIST/raw/t10k-images-idx3-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/1648877 [00:00<?, ?it/s]\u001b[A\n",
      "  1%|          | 16384/1648877 [00:00<00:17, 93269.63it/s]\u001b[A\n",
      "  2%|▏         | 40960/1648877 [00:00<00:14, 111483.41it/s]\u001b[A\n",
      "  4%|▍         | 73728/1648877 [00:00<00:11, 135547.73it/s]\u001b[A\n",
      "  6%|▋         | 106496/1648877 [00:00<00:09, 159667.68it/s]\u001b[A\n",
      "  8%|▊         | 139264/1648877 [00:00<00:08, 182406.18it/s]\u001b[A\n",
      " 10%|█         | 172032/1648877 [00:01<00:07, 202567.55it/s]\u001b[A\n",
      " 12%|█▏        | 204800/1648877 [00:01<00:06, 225034.04it/s]\u001b[A\n",
      " 15%|█▍        | 245760/1648877 [00:01<00:05, 248800.41it/s]\u001b[A\n",
      " 17%|█▋        | 286720/1648877 [00:01<00:05, 266601.63it/s]\u001b[A\n",
      " 20%|█▉        | 327680/1648877 [00:01<00:04, 287756.23it/s]\u001b[A\n",
      " 22%|██▏       | 368640/1648877 [00:01<00:04, 304658.38it/s]\u001b[A\n",
      " 25%|██▌       | 417792/1648877 [00:01<00:03, 319732.72it/s]\u001b[A\n",
      " 28%|██▊       | 466944/1648877 [00:01<00:03, 342249.75it/s]\u001b[A\n",
      " 31%|███▏      | 516096/1648877 [00:02<00:03, 360032.20it/s]\u001b[A\n",
      " 35%|███▍      | 573440/1648877 [00:02<00:02, 388787.75it/s]\u001b[A\n",
      " 38%|███▊      | 630784/1648877 [00:02<00:02, 411824.23it/s]\u001b[A\n",
      " 42%|████▏     | 688128/1648877 [00:02<00:02, 429632.84it/s]\u001b[A\n",
      " 45%|████▌     | 745472/1648877 [00:02<00:02, 443054.67it/s]\u001b[A\n",
      " 49%|████▊     | 802816/1648877 [00:02<00:02, 399887.43it/s]\u001b[A\n",
      " 54%|█████▍    | 892928/1648877 [00:02<00:01, 476937.21it/s]\u001b[A\n",
      " 58%|█████▊    | 950272/1648877 [00:02<00:01, 481951.23it/s]\u001b[A\n",
      " 61%|██████    | 1007616/1648877 [00:03<00:01, 480732.24it/s]\u001b[A\n",
      " 65%|██████▍   | 1064960/1648877 [00:03<00:01, 463646.53it/s]\u001b[A\n",
      " 68%|██████▊   | 1114112/1648877 [00:03<00:01, 463538.22it/s]\u001b[A\n",
      " 71%|███████   | 1163264/1648877 [00:03<00:01, 445937.61it/s]\u001b[A\n",
      " 74%|███████▎  | 1212416/1648877 [00:03<00:01, 307334.97it/s]\u001b[A\n",
      " 79%|███████▉  | 1310720/1648877 [00:03<00:00, 380403.66it/s]\u001b[A\n",
      " 83%|████████▎ | 1368064/1648877 [00:03<00:00, 397666.81it/s]\u001b[A\n",
      " 86%|████████▋ | 1425408/1648877 [00:04<00:00, 372988.08it/s]\u001b[A\n",
      " 89%|████████▉ | 1474560/1648877 [00:04<00:00, 383295.30it/s]\u001b[A\n",
      " 92%|█████████▏| 1523712/1648877 [00:04<00:00, 394608.66it/s]\u001b[A\n",
      " 95%|█████████▌| 1572864/1648877 [00:04<00:00, 308661.36it/s]\u001b[A\n",
      " 99%|█████████▉| 1630208/1648877 [00:04<00:00, 355701.73it/s]\u001b[A\n",
      "1654784it [00:04, 341097.69it/s]                             \u001b[A\n",
      "0it [00:00, ?it/s]\u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/t10k-images-idx3-ubyte.gz to data/MNIST/raw\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz to data/MNIST/raw/t10k-labels-idx1-ubyte.gz\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/4542 [00:00<?, ?it/s]\u001b[A\n",
      "8192it [00:00, 32527.68it/s]            \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting data/MNIST/raw/t10k-labels-idx1-ubyte.gz to data/MNIST/raw\n",
      "Processing...\n",
      "Done!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "9920512it [00:40, 465606.19it/s]                             "
     ]
    }
   ],
   "source": [
    "torch_model = SimpleCNN()\n",
    "\n",
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('data', train=True, download=True,\n",
    "                   transform=transforms.ToTensor()),\n",
    "    batch_size=batch_size, shuffle=True)\n",
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('data', train=False, transform=transforms.ToTensor()),\n",
    "    batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we are ready for training the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-14-f1893fa47165>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m     19\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     20\u001b[0m         \u001b[0;31m# Calculamos gradientes (backpropagation)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 21\u001b[0;31m         \u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     22\u001b[0m         \u001b[0mtrain_loss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mappend\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mloss\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mitem\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     23\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/torch/tensor.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(self, gradient, retain_graph, create_graph)\u001b[0m\n\u001b[1;32m    116\u001b[0m                 \u001b[0mproducts\u001b[0m\u001b[0;34m.\u001b[0m \u001b[0mDefaults\u001b[0m \u001b[0mto\u001b[0m\u001b[0;31m \u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;32mFalse\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;31m`\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    117\u001b[0m         \"\"\"\n\u001b[0;32m--> 118\u001b[0;31m         \u001b[0mtorch\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mautograd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbackward\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgradient\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    120\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mregister_hook\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mhook\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda3/lib/python3.7/site-packages/torch/autograd/__init__.py\u001b[0m in \u001b[0;36mbackward\u001b[0;34m(tensors, grad_tensors, retain_graph, create_graph, grad_variables)\u001b[0m\n\u001b[1;32m     91\u001b[0m     Variable._execution_engine.run_backward(\n\u001b[1;32m     92\u001b[0m         \u001b[0mtensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mgrad_tensors\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mretain_graph\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcreate_graph\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 93\u001b[0;31m         allow_unreachable=True)  # allow_unreachable flag\n\u001b[0m\u001b[1;32m     94\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     95\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "# Training\n",
    "optimizer = optim.Adam(torch_model.parameters(), lr=lr)\n",
    "train_loss = []\n",
    "\n",
    "total = 0\n",
    "correct = 0\n",
    "step = 0\n",
    "for epoch in range(n_epochs):\n",
    "    for xs, ys in train_loader:\n",
    "\n",
    "        optimizer.zero_grad()\n",
    "        \n",
    "        # Compute predictions\n",
    "        preds = torch_model(xs)\n",
    "        loss = F.nll_loss(preds, ys)\n",
    "        \n",
    "        # Compute gradients (backpropagation)\n",
    "        loss.backward()  \n",
    "        train_loss.append(loss.data.item())\n",
    "        \n",
    "        # Update parameters\n",
    "        optimizer.step()  \n",
    "\n",
    "        correct += (torch.argmax(preds, axis=1) == ys).sum()\n",
    "        total += len(xs)\n",
    "        step += 1\n",
    "        if total % 1000 == 0:\n",
    "            acc = float(correct) / total\n",
    "            print('[%s] Training accuracy: %.2f%%' % (step, acc * 100))\n",
    "            total = 0\n",
    "            correct = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# torch.save(torch_model.state_dict(), 'cnn_weights.bin')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For this tutorial, instead of waiting for training to finish, we will just load some pretrained weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<All keys matched successfully>"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "torch_model = SimpleCNN()\n",
    "torch_model.load_state_dict(torch.load('cnn_weights.bin'))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We evaluate over a clean test set (without attacking)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[9] Clean accuracy: 99.11%\n"
     ]
    }
   ],
   "source": [
    "total = 0\n",
    "correct = 0\n",
    "for xs, ys in test_loader:\n",
    "\n",
    "    preds = torch_model(xs)\n",
    "    preds_np = preds.data.cpu().numpy()\n",
    "\n",
    "    correct += (torch.argmax(preds, axis=1) == ys).sum()\n",
    "    total += len(xs)\n",
    "\n",
    "acc = float(correct) / total\n",
    "print('[%s] Clean accuracy: %.2f%%' % (step, acc * 100))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Evaluating our classifier under adversarial attacks"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we will study the famous attack *fast gradient sign method* (FGSM), one of the first successful attacks against deep neural networks. It was introduced by Goodfellow et al in 2014 https://arxiv.org/abs/1412.6572\n",
    "\n",
    "The main idea is to slightly perturb an original image $x$ following the sign of the gradient of the loss $\\mathcal{L}$:\n",
    "\n",
    "$$\n",
    "x' = x + \\epsilon \\mbox{sign}(\\nabla_x \\mathcal{L}(x,y,\\theta))\n",
    "$$"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will do it over an example"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f1ddb069da0>"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAN6klEQVR4nO3df6hc9ZnH8c+z2QbjbTDJRsMliWtb8odhQSsxLKwsWUriLyQGJDbgom7wFkyg4oIbXLHCUhHduqwi1VuiTaVrKZhsNAYaE+v6C4NXiRp127gaqdebe1eDxEa0ap79Y851b5I537k5P+ZM7vN+wWVmzjNz5mHIJ+fHd+Z8zd0FYOr7s6YbANAdhB0IgrADQRB2IAjCDgTx5918MzPj1D9QM3e3dstLbdnN7CIz+52ZvW1mG8qsC0C9rOg4u5lNk/R7ScslvS/pJUlr3P3NxGvYsgM1q2PLvlTS2+7+jrv/SdKvJK0ssT4ANSoT9vmS/jDh8fvZsqOY2YCZDZnZUIn3AlBS7Sfo3H1Q0qDEbjzQpDJb9mFJCyc8XpAtA9CDyoT9JUmLzOxbZjZd0vclPVZNWwCqVng33t2/NLP1kn4jaZqkB939jco6A1CpwkNvhd6MY3agdrV8qQbAyYOwA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIApP2YzuMWs7KefXFixYkFu79dZbk69du3ZtoZ7GdeotNUvwc889l3ztTTfdlKy/+OKLyTqOVirsZrZf0ieSvpL0pbsvqaIpANWrYsv+d+7+YQXrAVAjjtmBIMqG3SXtMLOXzWyg3RPMbMDMhsxsqOR7ASih7G78Be4+bGZnSHrSzP7b3Z+Z+AR3H5Q0KElmln+2BkCtSm3Z3X04ux2TtEXS0iqaAlC9wmE3sz4zmzl+X9IKSXuragxAtSw1Dpp8odm31dqaS63Dgf9w9x93eA278W2sWrUqWb/00kuT9WuvvbbKdnrGtm3bkvWVK1d2qZOTi7u3/fJD4WN2d39H0jmFOwLQVQy9AUEQdiAIwg4EQdiBIAg7EEThobdCbxZ06G316tXJ+saNG5P1U089tcp2TsjHH3+crJ9yyiml6imjo6PJ+ooVK5L1vXtjfu0jb+iNLTsQBGEHgiDsQBCEHQiCsANBEHYgCMIOBME4ewWWLk1fs2Pnzp3Jel9fX5XtnJDHH388Wb/llluS9U7fAdi+fXtubfbs2cnXdvLqq68m6+edd16p9Z+sGGcHgiPsQBCEHQiCsANBEHYgCMIOBEHYgSCYsrkCN954Y7Je9zj6Z599llsbGGg7K9fXOo2zHzp0qFBP41Jj4cuWLSu17lmzZiXr/f39ubWRkZFS730yYssOBEHYgSAIOxAEYQeCIOxAEIQdCIKwA0Ewzl6B008/vdb17969O1m/4oorcmsffPBB1e2ckOHh4drW3en7C6eddlpujXH2NszsQTMbM7O9E5bNMbMnzWxfdlvuKgQAajeZ3fifS7romGUbJO1y90WSdmWPAfSwjmF392ckHTxm8UpJm7L7myRdXnFfACpW9Jh9nruPH/QckDQv74lmNiAp/QVtALUrfYLO3T11IUl3H5Q0KE3dC04CJ4OiQ2+jZtYvSdntWHUtAahD0bA/Junq7P7VkrZW0w6AunS8bryZPSJpmaS5kkYl/UjSf0r6taQzJb0nabW7H3sSr926puRu/OLFi5P1K6+8stT677zzzmT98OHDpdZfp9R49hlnnFFq3Vw3vr2868Z3PGZ39zU5pe+V6ghAV/F1WSAIwg4EQdiBIAg7EARhB4JgymaUMmPGjGR9//79ubW5c+cmX3vkyJFk/emnn07Wly9fnqxPVUzZDARH2IEgCDsQBGEHgiDsQBCEHQiCsANBcClpJM2cOTNZf/jhh5P1TmPpKZ0uQx11HL0otuxAEIQdCIKwA0EQdiAIwg4EQdiBIAg7EATj7Ei66qqrkvXLLrus8Lo7/V79vvvuK7xuHI8tOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4EwXXjg1u1alWy/tBDDyXrnX7vnvLAAw8k69dff33hdUdW+LrxZvagmY2Z2d4Jy24zs2Ez25P9XVJlswCqN5nd+J9LuqjN8n9z93Ozv+3VtgWgah3D7u7PSDrYhV4A1KjMCbr1ZvZatps/O+9JZjZgZkNmNlTivQCUVDTsP5X0HUnnShqR9JO8J7r7oLsvcfclBd8LQAUKhd3dR939K3c/IulnkpZW2xaAqhUKu5n1T3i4StLevOcC6A0dx9nN7BFJyyTNlTQq6UfZ43MluaT9kn7g7iMd34xx9q6bNWtWsv7UU08l6+ecc06V7Rxl/vz5yfqBAwdqe++pLG+cvePFK9x9TZvFG0t3BKCr+LosEARhB4Ig7EAQhB0IgrADQXAp6Smgr68vt7Zly5bka+scWpOkgYGB3Nro6Git742jsWUHgiDsQBCEHQiCsANBEHYgCMIOBEHYgSC4lPQUcOaZZ+bW3n333S52crxFixbl1sbGxpKv/fzzz5P1L774olBPU13hS0kDmBoIOxAEYQeCIOxAEIQdCIKwA0EQdiAIfs9+EpgzZ06yvnXr1i51cuL27dtX+LVPPPFEsr5mTbsLH/+/w4cPF37vqYgtOxAEYQeCIOxAEIQdCIKwA0EQdiAIwg4Ewe/ZTwL33HNPsr5u3bouddJbOn2/4JprrsmtHTp0qOJuekfh37Ob2UIz+62ZvWlmb5jZD7Plc8zsSTPbl93OrrppANWZzG78l5L+0d0XS/prSevMbLGkDZJ2ufsiSbuyxwB6VMewu/uIu7+S3f9E0luS5ktaKWlT9rRNki6vq0kA5Z3Qd+PN7CxJ35W0W9I8dx/JSgckzct5zYCk/Am/AHTFpM/Gm9k3JT0q6QZ3P+rshrfO8rU9+ebug+6+xN2XlOoUQCmTCruZfUOtoP/S3Tdni0fNrD+r90tKXyoUQKM6Dr2Zmal1TH7Q3W+YsPwuSR+5+x1mtkHSHHe/qcO6GHprY/369cn63XffnaxPmzatynamjIsvvji3tmPHji520l15Q2+TOWb/G0l/L+l1M9uTLbtZ0h2Sfm1mayW9J2l1FY0CqEfHsLv7c5La/k8h6XvVtgOgLnxdFgiCsANBEHYgCMIOBEHYgSC4lHQXXHjhhcn6XXfdlazXOY7+wgsvJOvPPvtssr558+Zk/fzzz8+t3XvvvcnXtr7igaqwZQeCIOxAEIQdCIKwA0EQdiAIwg4EQdiBIBhn74ING9LX4pw+fXqp9X/66ae5tU6Xob799tuT9bLTHg8NDeXWzj777ORrO10i+6OPPkrWh4eHk/Vo2LIDQRB2IAjCDgRB2IEgCDsQBGEHgiDsQBBM2dwF999/f7J+3XXXJetHjhxJ1lPXR9+5c2fytU2aMWNGsr5s2bJk/fnnn0/Wp/K0zCmFp2wGMDUQdiAIwg4EQdiBIAg7EARhB4Ig7EAQk5mffaGkX0iaJ8klDbr7v5vZbZKuk/S/2VNvdvftHdYVcpwd6Ka8cfbJhL1fUr+7v2JmMyW9LOlyteZj/6O7/+tkmyDsQP3ywj6Z+dlHJI1k9z8xs7ckza+2PQB1O6FjdjM7S9J3Je3OFq03s9fM7EEzm53zmgEzGzKz/OsTAajdpL8bb2bflPRfkn7s7pvNbJ6kD9U6jv8XtXb1/6HDOtiNB2pW+JhdkszsG5K2SfqNu9/dpn6WpG3u/lcd1kPYgZoV/iGMtabS3CjprYlBz07cjVslaW/ZJgHUZzJn4y+Q9Kyk1yWN/9byZklrJJ2r1m78fkk/yE7mpdbFlh2oWand+KoQdqB+/J4dCI6wA0EQdiAIwg4EQdiBIAg7EARhB4Ig7EAQhB0IgrADQRB2IAjCDgRB2IEgCDsQRMcLTlbsQ0nvTXg8N1vWi3q1t17tS6K3oqrs7S/zCl39Pftxb2425O5LGmsgoVd769W+JHorqlu9sRsPBEHYgSCaDvtgw++f0qu99WpfEr0V1ZXeGj1mB9A9TW/ZAXQJYQeCaCTsZnaRmf3OzN42sw1N9JDHzPab2etmtqfp+emyOfTGzGzvhGVzzOxJM9uX3badY6+h3m4zs+Hss9tjZpc01NtCM/utmb1pZm+Y2Q+z5Y1+dom+uvK5df2Y3cymSfq9pOWS3pf0kqQ17v5mVxvJYWb7JS1x98a/gGFmfyvpj5J+MT61lpndKemgu9+R/Uc5293/qUd6u00nOI13Tb3lTTN+jRr87Kqc/ryIJrbsSyW97e7vuPufJP1K0soG+uh57v6MpIPHLF4paVN2f5Na/1i6Lqe3nuDuI+7+Snb/E0nj04w3+tkl+uqKJsI+X9IfJjx+X70137tL2mFmL5vZQNPNtDFvwjRbByTNa7KZNjpO491Nx0wz3jOfXZHpz8viBN3xLnD38yRdLGldtrvak7x1DNZLY6c/lfQdteYAHJH0kyabyaYZf1TSDe5+aGKtyc+uTV9d+dyaCPuwpIUTHi/IlvUEdx/ObsckbVHrsKOXjI7PoJvdjjXcz9fcfdTdv3L3I5J+pgY/u2ya8Ucl/dLdN2eLG//s2vXVrc+tibC/JGmRmX3LzKZL+r6kxxro4zhm1pedOJGZ9Ulaod6bivoxSVdn96+WtLXBXo7SK9N4500zroY/u8anP3f3rv9JukStM/L/I+mfm+ghp69vS3o1+3uj6d4kPaLWbt0Xap3bWCvpLyTtkrRP0k5Jc3qot4fVmtr7NbWC1d9QbxeotYv+mqQ92d8lTX92ib668rnxdVkgCE7QAUEQdiAIwg4EQdiBIAg7EARhB4Ig7EAQ/wdyDHXI3OBtOwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "idx = 1\n",
    "sample = xs[idx]\n",
    "plt.imshow(sample.numpy()[0,:,:], cmap='gray')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted class: 2\n"
     ]
    }
   ],
   "source": [
    "sample.requires_grad = True\n",
    "output = torch_model(sample.unsqueeze(0))\n",
    "print('Predicted class:', torch.argmax( output ).numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The previous example was correctly classified. Let's see what happens under the FGSM attack"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Gradient computation\n",
    "F.nll_loss(output, ys[idx].unsqueeze(0)).backward()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we have just computed $\\nabla_x \\mathcal{L}(x,y,\\theta)$:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "torch.Size([1, 28, 28])"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sample.grad.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will apply the FGSM attack:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "adv_sample = sample + 0.1*torch.sign(sample.grad)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Predicted class: 7\n"
     ]
    }
   ],
   "source": [
    "adv_output = torch_model(adv_sample.unsqueeze(0))\n",
    "print('Predicted class:', torch.argmax( adv_output ).numpy())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "It seems we have fool our classifier. What is the visual aspect of the attacked input?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x7f1ddabc55f8>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD4CAYAAAAq5pAIAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4xLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy8QZhcZAAAP3klEQVR4nO3dX4wVZZrH8d9jCzY4RGFdseOwO/REY4hxmU1DTPwTNpMl4g2iiRkMG0yIPUZMIA5x/RMdb0zMujrhQif2LGSYzSzjJDOOXJDdYRGjExMQCCCgrthBB2xowAsgQfn37EUXpsVTbzXnPXWqut/vJ+n06XpOVT2U/fOcrvdUvebuAjD2XVZ1AwDag7ADiSDsQCIIO5AIwg4k4vJ27szMok79d3Z2tqqV7/jqq69K23aRon9XUW9j9bgUiTlusce8SJn/TULOnDmjs2fPWqNaVNjN7C5JKyV1SPoPd38hZntFuru7S9v23r17S9t2kaJ/V1FvY/W4FIk5brHHvEiZ/01C+vv7c2tNv403sw5Jr0iaJ2mGpIVmNqPZ7QEoV8zf7LMl7XP3fnc/Lel3kua3pi0ArRYT9usl/XXYzweyZd9iZr1mttXMtkbsC0Ck0k/QuXufpD4p/gQdgObFvLIflDRt2M/fz5YBqKGYsL8v6QYzm25m4yX9RNK61rQFoNWafhvv7mfN7FFJ/6OhobfV7r4nppkZM8In80PDIUXrlil237HDPHU9LmWrclgw5ne1qm1H/c3u7uslrY/ZBoD24OOyQCIIO5AIwg4kgrADiSDsQCIIO5CItl7PHqvMMeOxPB4dUjRmW+VxKXMcvepLd2OOa2jdUi5xBTC6EHYgEYQdSARhBxJB2IFEEHYgEdbOiR2rvFPNWB1CKttoHpIczcc9hrs3vJU0r+xAIgg7kAjCDiSCsAOJIOxAIgg7kAjCDiSiVuPsVd5Kusxb/5ZttI4nl33c6nxcQv/22L4ZZwcSR9iBRBB2IBGEHUgEYQcSQdiBRBB2IBG1GmcvU9Vj4THMGg6bfuPYsWO5tUceeSS47n333ddUTxcU9Rb6/dq+fXtw3fXrwxME79y5M1gPGa1j8FJx73nj7FH3jTez/ZJOSDon6ay798RsD0B5WjFJxD+5+9EWbAdAifibHUhEbNhd0p/NbJuZ9TZ6gpn1mtlWM9sauS8AEWLfxt/u7gfN7FpJG8zsI3d/Z/gT3L1PUp9U7Qk6IHVRr+zufjD7PijpDUmzW9EUgNZrOuxmdqWZTbrwWNJcSbtb1RiA1op5Gz9V0hvZOOvlkv7L3f87tEJnZ6e6u7sjdpmvzuOmRebOnRus33nnncH6ggULmt53C66dbnrdRYsWBevTpk0L1os+Q1Dn34kqpmxuOuzu3i/pH5pdH0B7MfQGJIKwA4kg7EAiCDuQCMIOJKIVF8LUQp0vYZ03b16w/vzzzwfrn376abBe5m2wT5w4EaxfccUVwfr48eMvuacLbr755mD9xhtvbHrbRYqOaexlqGXeFj0Pr+xAIgg7kAjCDiSCsAOJIOxAIgg7kAjCDiSirbeSnjBhgpd1iWuVbrnllmB99erVwfqECRNa2c4lefvtt4P1lStXBuudnZ3B+muvvZZbO3DgQHDdIh0dHcH6vffeG7X90ai/v1+nTp1iymYgZYQdSARhBxJB2IFEEHYgEYQdSARhBxJRq+vZq7z1b8w1xA8++GCwXvY4+unTp3Nrzz77bHDdTZs2BesnT55sqqcLPv7449zarFmzgusW/T58/vnnwfq1116bWxscHAyuOxbxyg4kgrADiSDsQCIIO5AIwg4kgrADiSDsQCJqNc4eey/umG3HmDJlSmnblqS1a9cG6319fbm1ovHksj/b8NZbb+XWisbZixR9fmHSpEm5taqPS0hZOSh8ZTez1WY2aGa7hy2bYmYbzOyT7PvkpvYOoG1G8jb+15LuumjZE5I2uvsNkjZmPwOoscKwu/s7kr68aPF8SWuyx2sk3dPivgC0WLN/s09194Hs8SFJU/OeaGa9knolady4cU3uDkCs6LPxPnTHyty7Vrp7n7v3uHtP0Q0CAZSn2bAfNrMuScq+p3cJETDKNBv2dZIWZ48XS3qzNe0AKEvhfePNbK2kOZKukXRY0s8l/UnS7yX9naTPJN3v7hefxGu0rfbdpL7FQmOfX3/9dXDdFStWBOt79uwJ1letWhWsnzp1KliPETvme+zYsdzaoUOHmurpgo8++ihYf+aZZ3JrZc+/XqZQb6H7xheeoHP3hTmlH4+sNQB1wMdlgUQQdiARhB1IBGEHEkHYgUS0dcrm0Tz0lqqiIaiiKZs3btyYW/viiy+C654/fz5Y37JlS7C+ZMmSYH2scnembAZSRtiBRBB2IBGEHUgEYQcSQdiBRBB2IBFtvZV0Z2enuru7c+tVXjaIxiZOnBisv/jii8H61VdfnVsrGmc/fPhwsF40jl7m7cPrfIlrHl7ZgUQQdiARhB1IBGEHEkHYgUQQdiARhB1IxKiasjmk7FsDVzlmG9N7bN/z588P1ufMmdP0touuV9+5c2ewXubvS52VNmUzgLGBsAOJIOxAIgg7kAjCDiSCsAOJIOxAImo1zl7lWHeVY9lVjuEvX748WH/sscdK239Rb0VTVceo85TMZSl8ZTez1WY2aGa7hy17zswOmtmO7OvuctsEEGskb+N/LemuBst/4e4zs6/1rW0LQKsVht3d35H0ZRt6AVCimBN0j5rZruxt/uS8J5lZr5ltNbOt586di9gdgBjNhv2Xkn4oaaakAUkv5T3R3fvcvcfdezo6OprcHYBYTYXd3Q+7+zl3Py/pV5Jmt7YtAK3WVNjNrGvYjwsk7c57LoB6KBxnN7O1kuZIusbMDkj6uaQ5ZjZTkkvaL+mnrWgmZry5zLHqdmw/JGbM99Zbbw3W77jjjmC96L7xMV599dXStl1kLI6jFykMu7svbLC4vE87ACgFH5cFEkHYgUQQdiARhB1IBGEHEjFmLnGt8vLYshX1Hhoee/jhh4Pr3nTTTU31NFKvv/56bu3o0aNR2x6tl0QXrR+zLlM2AyDsQCoIO5AIwg4kgrADiSDsQCIIO5CIWo2zl3k757E6JbMkdXV15dZmzZoVte8iRb1t3rw5tzZhwoTgumfOnAnWq7wddJWXyDJlM4Agwg4kgrADiSDsQCIIO5AIwg4kgrADiajVOHuq17MX9X7VVVcF66+88kor2/mW2PHkl19+uel1jxw5EqyvWLEiWB+tt/8uC6/sQCIIO5AIwg4kgrADiSDsQCIIO5AIwg4kwty9fTsza9/OxpAnn3wyWF+0aFFureyx5irHkw8dOhSsh47bli1bgutWea18LHe3RssLX9nNbJqZbTKzvWa2x8yWZcunmNkGM/sk+z651U0DaJ2RvI0/K+ln7j5D0q2SlprZDElPSNro7jdI2pj9DKCmCsPu7gPuvj17fELSh5KulzRf0prsaWsk3VNWkwDiXdJn483sB5J+JGmzpKnuPpCVDkmamrNOr6Te5lsE0AojPhtvZt+T9AdJy939+PCaD53la3jyzd373L3H3XuiOgUQZURhN7NxGgr6b939j9niw2bWldW7JA2W0yKAVih8G29mJmmVpA/dffj1iuskLZb0Qvb9zaJtdXZ2qru7O7de5+GMMi1cuDCqHjO8NpqP+XXXXResX35581dwj+bjkmckR+M2Sf8i6QMz25Ete0pDIf+9mS2R9Jmk+8tpEUArFIbd3f8iqeEgvaQft7YdAGXh47JAIgg7kAjCDiSCsAOJIOxAImp1K+kiMbeSrtJtt90WrD/++OPBekdHR9P7Ljoup0+fDtbXrFkTrA8MDATrQx/TaOzpp58OrnvZZbwWNRLKQX9/f26NowkkgrADiSDsQCIIO5AIwg4kgrADiSDsQCJqNc4ec/ve2Fv/lnnr4IceeihYnzlzZtPblqTt27fn1rZt2xZcd9myZcH69OnTm+ppJHbt2hWsP/DAA8H6e++9F6wPDtb3fioxnxlp9neRV3YgEYQdSARhBxJB2IFEEHYgEYQdSARhBxJRq3H2MsVOXRyz/sSJE6P2XTSt9tKlS3Nrx48fz61J5Y6jF3nppZeC9XfffTdYD32+QJL27dt3yT21SxX3X+CVHUgEYQcSQdiBRBB2IBGEHUgEYQcSQdiBRFjRGK6ZTZP0G0lTJbmkPndfaWbPSXpI0pHsqU+5+/qCbQV3Vub17ChH7H0CqhI7zl3m/Q9iuXvDm/WP5EM1ZyX9zN23m9kkSdvMbENW+4W7/3urmgRQnpHMzz4gaSB7fMLMPpR0fdmNAWitS/qb3cx+IOlHkjZnix41s11mttrMJues02tmW81sa1SnAKKMOOxm9j1Jf5C03N2PS/qlpB9KmqmhV/6GH3R29z5373H3nhb0C6BJIwq7mY3TUNB/6+5/lCR3P+zu59z9vKRfSZpdXpsAYhWG3Yam4Vwl6UN3f3nY8q5hT1sgaXfr2wPQKiMZertd0ruSPpB0Plv8lKSFGnoL75L2S/ppdjIvtK3wzkapug4vtUKVQ0ij+bjGDEnGrNvf369Tp041N/Tm7n+R1Gjl4Jg6gHrhE3RAIgg7kAjCDiSCsAOJIOxAIgg7kIjCcfaW7myMjrPHih1PrnIsvExlTsOd4iWuvLIDiSDsQCIIO5AIwg4kgrADiSDsQCIIO5CIdo+zH5H02bBF10g62rYGLk1de6trXxK9NauVvf29u/9to0Jbw/6dnZttreu96eraW137kuitWe3qjbfxQCIIO5CIqsPeV/H+Q+raW137kuitWW3prdK/2QG0T9Wv7ADahLADiagk7GZ2l5l9bGb7zOyJKnrIY2b7zewDM9tR9fx02Rx6g2a2e9iyKWa2wcw+yb43nGOvot6eM7OD2bHbYWZ3V9TbNDPbZGZ7zWyPmS3Llld67AJ9teW4tf1vdjPrkPR/kv5Z0gFJ70ta6O61uAODme2X1OPulX8Aw8zulHRS0m/c/eZs2b9J+tLdX8j+RznZ3f+1Jr09J+lk1dN4Z7MVdQ2fZlzSPZIeVIXHLtDX/WrDcavilX22pH3u3u/upyX9TtL8CvqoPXd/R9KXFy2eL2lN9niNhn5Z2i6nt1pw9wF33549PiHpwjTjlR67QF9tUUXYr5f012E/H1C95nt3SX82s21m1lt1Mw1MHTbN1iFJU6tspoHCabzb6aJpxmtz7JqZ/jwWJ+i+63Z3/0dJ8yQtzd6u1pIP/Q1Wp7HTEU3j3S4Nphn/RpXHrtnpz2NVEfaDkqYN+/n72bJacPeD2fdBSW+oflNRH74wg272fbDifr5Rp2m8G00zrhocuyqnP68i7O9LusHMppvZeEk/kbSugj6+w8yuzE6cyMyulDRX9ZuKep2kxdnjxZLerLCXb6nLNN5504yr4mNX+fTn7t72L0l3a+iM/KeSnq6ih5y+uiXtzL72VN2bpLUaelt3RkPnNpZI+htJGyV9Iul/JU2pUW//qaGpvXdpKFhdFfV2u4beou+StCP7urvqYxfoqy3HjY/LAongBB2QCMIOJIKwA4kg7EAiCDuQCMIOJIKwA4n4f/48nv7h47UiAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.imshow(adv_sample.detach().numpy()[0,:,:], cmap='gray')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This doesn't look as a 7..."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Can you compute the accuracy over the whole attacked test set?"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}