Maxout Networks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np \n",
    "import torch \n",
    "import torch.nn as nn \n",
    "from torchvision import datasets, transforms \n",
    "from torch.nn import init\n",
    "import torch.nn.functional as F\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns\n",
    "%matplotlib inline"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "copy from [Maxout Networks](https://github.com/Duncanswilson/maxout-pytorch/blob/master/maxout_pytorch.ipynb)\n",
    "<br>\n",
    "pytorch 0.4.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.0001\n",
    "batch_size = 50\n",
    "n_epochs = 3\n",
    "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading http://yann.lecun.com/exdb/mnist/train-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/train-labels-idx1-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-images-idx3-ubyte.gz\n",
      "Downloading http://yann.lecun.com/exdb/mnist/t10k-labels-idx1-ubyte.gz\n",
      "Processing...\n",
      "Done!\n"
     ]
    }
   ],
   "source": [
    "train_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('./data', train=True, download=True,\n",
    "                   transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "                       batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_loader = torch.utils.data.DataLoader(\n",
    "    datasets.MNIST('./data', train=False, transform=transforms.Compose([\n",
    "                       transforms.ToTensor(),\n",
    "                       transforms.Normalize((0.1307,), (0.3081,))\n",
    "                       ])),\n",
    "                       batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class rectifier_mlp(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(rectifier_mlp, self).__init__()\n",
    "        self.fc1 = nn.Linear(784, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 10)\n",
    "        self.dropout = nn.Dropout()\n",
    "\n",
    "    def forward(self, x): \n",
    "        x = x.view(-1, 784)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class maxout_mlp(nn.Module):\n",
    "    def __init__(self, num_units=2):\n",
    "        super(maxout_mlp, self).__init__()\n",
    "        self.fc1_list = nn.ModuleList()\n",
    "        self.fc2_list = nn.ModuleList()\n",
    "        self.dropout = nn.Dropout()\n",
    "        for _ in range(num_units):\n",
    "            self.fc1_list.append(nn.Linear(784, 1024))\n",
    "            self.fc2_list.append(nn.Linear(1024, 10))\n",
    "\n",
    "    def forward(self, x): \n",
    "        x = x.view(-1, 784)\n",
    "        x = self.maxout(x, self.fc1_list)\n",
    "        x = self.dropout(x)\n",
    "        x = self.maxout(x, self.fc2_list)\n",
    "        return x\n",
    "\n",
    "    def maxout(self, x, layer_list):\n",
    "        max_output = layer_list[0](x)\n",
    "        for _, layer in enumerate(layer_list, start=1):\n",
    "            max_output = torch.max(max_output, layer(x))\n",
    "        return max_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def train(epoch,  net, train_loss, train_acc): \n",
    "    optimizer = torch.optim.Adam(net.parameters(), lr=learning_rate)\n",
    "    net.train()\n",
    "    for batch_idx, (data, target) in enumerate(train_loader): \n",
    "        data, target = data.to(device), target.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        output = net(data)\n",
    "        loss = F.cross_entropy(output, target)\n",
    "        train_loss.append(loss.item())\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        prediction = output.data.max(1)[1]\n",
    "        accuracy = prediction.eq(target.data).sum().float()/batch_size*100.0 \n",
    "        train_acc.append(accuracy)\n",
    "        if batch_idx % 100 == 0:\n",
    "            print('Train Epoch: {} [{}/{} ({:.0f}%)]\\tLoss: {:.6f}'.format(\n",
    "                epoch, batch_idx * len(data), len(train_loader.dataset),\n",
    "                100. * batch_idx / len(train_loader), loss.item()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def test(epoch, net):\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct = 0\n",
    "    with torch.no_grad():\n",
    "        for data, target in test_loader:\n",
    "            data, target = data.to(device), target.to(device)\n",
    "            output = net(data)\n",
    "            test_loss += F.cross_entropy(output, target).item()\n",
    "            pred = output.data.max(1)[1] # get the index of the max log-probability\n",
    "            correct += pred.eq(target.data).cpu().sum()\n",
    "\n",
    "        test_loss = test_loss\n",
    "        test_loss /= len(test_loader) # loss function already averages over batch size\n",
    "        print('\\nTest set: Average loss: {:.4f}, Accuracy: {}/{} ({:.0f}%)\\n'.format(\n",
    "            test_loss, correct, len(test_loader.dataset),\n",
    "            100. * correct / len(test_loader.dataset)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.333392\n",
      "Train Epoch: 0 [5000/60000 (8%)]\tLoss: 0.668069\n",
      "Train Epoch: 0 [10000/60000 (17%)]\tLoss: 0.383142\n",
      "Train Epoch: 0 [15000/60000 (25%)]\tLoss: 0.519796\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.250225\n",
      "Train Epoch: 0 [25000/60000 (42%)]\tLoss: 0.424916\n",
      "Train Epoch: 0 [30000/60000 (50%)]\tLoss: 0.197328\n",
      "Train Epoch: 0 [35000/60000 (58%)]\tLoss: 0.400900\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.408135\n",
      "Train Epoch: 0 [45000/60000 (75%)]\tLoss: 0.160900\n",
      "Train Epoch: 0 [50000/60000 (83%)]\tLoss: 0.280716\n",
      "Train Epoch: 0 [55000/60000 (92%)]\tLoss: 0.285703\n",
      "\n",
      "Test set: Average loss: 0.2135, Accuracy: 9393/10000 (93%)\n",
      "\n",
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 0.123724\n",
      "Train Epoch: 1 [5000/60000 (8%)]\tLoss: 0.132765\n",
      "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 0.170325\n",
      "Train Epoch: 1 [15000/60000 (25%)]\tLoss: 0.330656\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.170931\n",
      "Train Epoch: 1 [25000/60000 (42%)]\tLoss: 0.206825\n",
      "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 0.259060\n",
      "Train Epoch: 1 [35000/60000 (58%)]\tLoss: 0.106665\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.360179\n",
      "Train Epoch: 1 [45000/60000 (75%)]\tLoss: 0.225617\n",
      "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 0.086727\n",
      "Train Epoch: 1 [55000/60000 (92%)]\tLoss: 0.066806\n",
      "\n",
      "Test set: Average loss: 0.1486, Accuracy: 9573/10000 (95%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.157338\n",
      "Train Epoch: 2 [5000/60000 (8%)]\tLoss: 0.235681\n",
      "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 0.125065\n",
      "Train Epoch: 2 [15000/60000 (25%)]\tLoss: 0.106488\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.311100\n",
      "Train Epoch: 2 [25000/60000 (42%)]\tLoss: 0.124668\n",
      "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 0.046461\n",
      "Train Epoch: 2 [35000/60000 (58%)]\tLoss: 0.331742\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.075372\n",
      "Train Epoch: 2 [45000/60000 (75%)]\tLoss: 0.088107\n",
      "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 0.147185\n",
      "Train Epoch: 2 [55000/60000 (92%)]\tLoss: 0.133448\n",
      "\n",
      "Test set: Average loss: 0.1171, Accuracy: 9658/10000 (96%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mlp_relu = rectifier_mlp().to(device)\n",
    "relu_mlp_loss = [] \n",
    "relu_mlp_acc = []\n",
    "for epoch in range(n_epochs): \n",
    "    train(epoch, mlp_relu, relu_mlp_loss, relu_mlp_acc)\n",
    "    test(epoch, mlp_relu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.341381\n",
      "Train Epoch: 0 [5000/60000 (8%)]\tLoss: 0.765297\n",
      "Train Epoch: 0 [10000/60000 (17%)]\tLoss: 0.424273\n",
      "Train Epoch: 0 [15000/60000 (25%)]\tLoss: 0.275276\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.377032\n",
      "Train Epoch: 0 [25000/60000 (42%)]\tLoss: 0.338851\n",
      "Train Epoch: 0 [30000/60000 (50%)]\tLoss: 0.155935\n",
      "Train Epoch: 0 [35000/60000 (58%)]\tLoss: 0.227752\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.123138\n",
      "Train Epoch: 0 [45000/60000 (75%)]\tLoss: 0.145293\n",
      "Train Epoch: 0 [50000/60000 (83%)]\tLoss: 0.202979\n",
      "Train Epoch: 0 [55000/60000 (92%)]\tLoss: 0.250366\n",
      "\n",
      "Test set: Average loss: 0.1365, Accuracy: 9624/10000 (96%)\n",
      "\n",
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 0.139623\n",
      "Train Epoch: 1 [5000/60000 (8%)]\tLoss: 0.052591\n",
      "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 0.043543\n",
      "Train Epoch: 1 [15000/60000 (25%)]\tLoss: 0.096602\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.054987\n",
      "Train Epoch: 1 [25000/60000 (42%)]\tLoss: 0.101092\n",
      "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 0.051419\n",
      "Train Epoch: 1 [35000/60000 (58%)]\tLoss: 0.129910\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.043604\n",
      "Train Epoch: 1 [45000/60000 (75%)]\tLoss: 0.217555\n",
      "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 0.116325\n",
      "Train Epoch: 1 [55000/60000 (92%)]\tLoss: 0.062521\n",
      "\n",
      "Test set: Average loss: 0.0925, Accuracy: 9725/10000 (97%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.082026\n",
      "Train Epoch: 2 [5000/60000 (8%)]\tLoss: 0.016901\n",
      "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 0.178813\n",
      "Train Epoch: 2 [15000/60000 (25%)]\tLoss: 0.135800\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.057042\n",
      "Train Epoch: 2 [25000/60000 (42%)]\tLoss: 0.081587\n",
      "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 0.031173\n",
      "Train Epoch: 2 [35000/60000 (58%)]\tLoss: 0.079199\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.079087\n",
      "Train Epoch: 2 [45000/60000 (75%)]\tLoss: 0.182722\n",
      "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 0.043531\n",
      "Train Epoch: 2 [55000/60000 (92%)]\tLoss: 0.018403\n",
      "\n",
      "Test set: Average loss: 0.0716, Accuracy: 9780/10000 (97%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "mlp_maxout = maxout_mlp(num_units=5).to(device)\n",
    "maxout_mlp_loss = [] \n",
    "maxout_mlp_acc = []\n",
    "for epoch in range(n_epochs): \n",
    "    train(epoch, mlp_maxout, maxout_mlp_loss, maxout_mlp_acc)\n",
    "    test(epoch, mlp_maxout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class rectifier_conv_net(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(rectifier_conv_net, self).__init__()\n",
    "        self.conv1 = nn.Conv2d(1, 32, 5, padding=2)  # zero-padding added to both side of the input\n",
    "        self.conv2 = nn.Conv2d(32, 64, 5, padding=2)\n",
    "        self.fc1 = nn.Linear(64*7*7, 1024)\n",
    "        self.fc2 = nn.Linear(1024, 10)\n",
    "        self.dropout = nn.Dropout()\n",
    "\n",
    "    def forward(self, x): \n",
    "        x = F.max_pool2d(F.relu(self.conv1(x)), 2)\n",
    "        # [B, 32, 14, 14]\n",
    "        x = F.max_pool2d(F.relu(self.conv2(x)), 2)\n",
    "        # [B, 64, 7, 7]\n",
    "        x = x.view(-1, 64*7*7)\n",
    "        x = F.relu(self.fc1(x))\n",
    "        x = self.dropout(x)\n",
    "        x = self.fc2(x)\n",
    "        return x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class maxout_conv_net(nn.Module):\n",
    "    def __init__(self, num_units=2):\n",
    "        super(maxout_conv_net, self).__init__()\n",
    "        self.conv1_list = nn.ModuleList()\n",
    "        self.conv2_list = nn.ModuleList()\n",
    "        self.fc1_list = nn.ModuleList()\n",
    "        self.fc2_list = nn.ModuleList()\n",
    "        self.dropout = nn.Dropout()\n",
    "        for _ in range(num_units):\n",
    "            self.conv1_list.append(nn.Conv2d(1, 32, 5, padding=2))\n",
    "            self.conv2_list.append(nn.Conv2d(32, 64, 5, padding=2))\n",
    "            self.fc1_list.append(nn.Linear(64*7*7, 1024))\n",
    "            self.fc2_list.append(nn.Linear(1024, 10))\n",
    "\n",
    "    def forward(self, x): \n",
    "        x = F.max_pool2d(self.maxout(x, self.conv1_list), 2)\n",
    "        x = F.max_pool2d(self.maxout(x, self.conv2_list), 2)\n",
    "        x = x.view(-1, 64*7*7)\n",
    "        x = self.maxout(x, self.fc1_list)\n",
    "        x = self.dropout(x)\n",
    "        x = self.maxout(x, self.fc2_list)\n",
    "        return x\n",
    "\n",
    "    def maxout(self, x, layer_list):\n",
    "        max_output = layer_list[0](x)\n",
    "        for _, layer in enumerate(layer_list, start=1):\n",
    "            max_output = torch.max(max_output, layer(x))\n",
    "        return max_output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.312882\n",
      "Train Epoch: 0 [5000/60000 (8%)]\tLoss: 0.543504\n",
      "Train Epoch: 0 [10000/60000 (17%)]\tLoss: 0.199146\n",
      "Train Epoch: 0 [15000/60000 (25%)]\tLoss: 0.355826\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.198580\n",
      "Train Epoch: 0 [25000/60000 (42%)]\tLoss: 0.113778\n",
      "Train Epoch: 0 [30000/60000 (50%)]\tLoss: 0.117420\n",
      "Train Epoch: 0 [35000/60000 (58%)]\tLoss: 0.207189\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.060757\n",
      "Train Epoch: 0 [45000/60000 (75%)]\tLoss: 0.302293\n",
      "Train Epoch: 0 [50000/60000 (83%)]\tLoss: 0.109785\n",
      "Train Epoch: 0 [55000/60000 (92%)]\tLoss: 0.031707\n",
      "\n",
      "Test set: Average loss: 0.0743, Accuracy: 9773/10000 (97%)\n",
      "\n",
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 0.088204\n",
      "Train Epoch: 1 [5000/60000 (8%)]\tLoss: 0.112422\n",
      "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 0.048545\n",
      "Train Epoch: 1 [15000/60000 (25%)]\tLoss: 0.110222\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.078297\n",
      "Train Epoch: 1 [25000/60000 (42%)]\tLoss: 0.037116\n",
      "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 0.026240\n",
      "Train Epoch: 1 [35000/60000 (58%)]\tLoss: 0.047568\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.019530\n",
      "Train Epoch: 1 [45000/60000 (75%)]\tLoss: 0.006378\n",
      "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 0.051312\n",
      "Train Epoch: 1 [55000/60000 (92%)]\tLoss: 0.085533\n",
      "\n",
      "Test set: Average loss: 0.0409, Accuracy: 9875/10000 (98%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.037899\n",
      "Train Epoch: 2 [5000/60000 (8%)]\tLoss: 0.080243\n",
      "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 0.018451\n",
      "Train Epoch: 2 [15000/60000 (25%)]\tLoss: 0.068353\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.043690\n",
      "Train Epoch: 2 [25000/60000 (42%)]\tLoss: 0.101088\n",
      "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 0.043519\n",
      "Train Epoch: 2 [35000/60000 (58%)]\tLoss: 0.009315\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.007277\n",
      "Train Epoch: 2 [45000/60000 (75%)]\tLoss: 0.112809\n",
      "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 0.012027\n",
      "Train Epoch: 2 [55000/60000 (92%)]\tLoss: 0.045477\n",
      "\n",
      "Test set: Average loss: 0.0352, Accuracy: 9894/10000 (98%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "net_relu = rectifier_conv_net().to(device)\n",
    "relu_loss = [] \n",
    "relu_acc = []\n",
    "for epoch in range(n_epochs): \n",
    "    train(epoch, net_relu, relu_loss, relu_acc)\n",
    "    test(epoch, net_relu)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train Epoch: 0 [0/60000 (0%)]\tLoss: 2.360562\n",
      "Train Epoch: 0 [5000/60000 (8%)]\tLoss: 0.295352\n",
      "Train Epoch: 0 [10000/60000 (17%)]\tLoss: 0.196501\n",
      "Train Epoch: 0 [15000/60000 (25%)]\tLoss: 0.160406\n",
      "Train Epoch: 0 [20000/60000 (33%)]\tLoss: 0.092157\n",
      "Train Epoch: 0 [25000/60000 (42%)]\tLoss: 0.029004\n",
      "Train Epoch: 0 [30000/60000 (50%)]\tLoss: 0.055983\n",
      "Train Epoch: 0 [35000/60000 (58%)]\tLoss: 0.080423\n",
      "Train Epoch: 0 [40000/60000 (67%)]\tLoss: 0.043930\n",
      "Train Epoch: 0 [45000/60000 (75%)]\tLoss: 0.051483\n",
      "Train Epoch: 0 [50000/60000 (83%)]\tLoss: 0.172550\n",
      "Train Epoch: 0 [55000/60000 (92%)]\tLoss: 0.166261\n",
      "\n",
      "Test set: Average loss: 0.0424, Accuracy: 9862/10000 (98%)\n",
      "\n",
      "Train Epoch: 1 [0/60000 (0%)]\tLoss: 0.072736\n",
      "Train Epoch: 1 [5000/60000 (8%)]\tLoss: 0.061517\n",
      "Train Epoch: 1 [10000/60000 (17%)]\tLoss: 0.255888\n",
      "Train Epoch: 1 [15000/60000 (25%)]\tLoss: 0.147335\n",
      "Train Epoch: 1 [20000/60000 (33%)]\tLoss: 0.071280\n",
      "Train Epoch: 1 [25000/60000 (42%)]\tLoss: 0.007710\n",
      "Train Epoch: 1 [30000/60000 (50%)]\tLoss: 0.087302\n",
      "Train Epoch: 1 [35000/60000 (58%)]\tLoss: 0.003828\n",
      "Train Epoch: 1 [40000/60000 (67%)]\tLoss: 0.041275\n",
      "Train Epoch: 1 [45000/60000 (75%)]\tLoss: 0.014343\n",
      "Train Epoch: 1 [50000/60000 (83%)]\tLoss: 0.033629\n",
      "Train Epoch: 1 [55000/60000 (92%)]\tLoss: 0.119920\n",
      "\n",
      "Test set: Average loss: 0.0279, Accuracy: 9912/10000 (99%)\n",
      "\n",
      "Train Epoch: 2 [0/60000 (0%)]\tLoss: 0.059666\n",
      "Train Epoch: 2 [5000/60000 (8%)]\tLoss: 0.047653\n",
      "Train Epoch: 2 [10000/60000 (17%)]\tLoss: 0.053885\n",
      "Train Epoch: 2 [15000/60000 (25%)]\tLoss: 0.153307\n",
      "Train Epoch: 2 [20000/60000 (33%)]\tLoss: 0.004085\n",
      "Train Epoch: 2 [25000/60000 (42%)]\tLoss: 0.070746\n",
      "Train Epoch: 2 [30000/60000 (50%)]\tLoss: 0.006293\n",
      "Train Epoch: 2 [35000/60000 (58%)]\tLoss: 0.007998\n",
      "Train Epoch: 2 [40000/60000 (67%)]\tLoss: 0.005290\n",
      "Train Epoch: 2 [45000/60000 (75%)]\tLoss: 0.001648\n",
      "Train Epoch: 2 [50000/60000 (83%)]\tLoss: 0.086402\n",
      "Train Epoch: 2 [55000/60000 (92%)]\tLoss: 0.050674\n",
      "\n",
      "Test set: Average loss: 0.0365, Accuracy: 9875/10000 (98%)\n",
      "\n"
     ]
    }
   ],
   "source": [
    "net_maxout = maxout_conv_net(num_units=5).to(device)\n",
    "maxout_loss = [] \n",
    "maxout_acc = []\n",
    "for epoch in range(n_epochs): \n",
    "    train(epoch, net_maxout, maxout_loss, maxout_acc)\n",
    "    test(epoch, net_maxout)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fbe27e74898>"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3Xl8FOX9B/DP7G7uhBAgF0cCIZEjHFEOBRQkNKBCBBFp\naVFL9aftT6VWW+RoEVBAraX+EA9AREVFRCupIsodkDuEEMKRhJCb3Nnce848vz82e2XPJJvske/7\n9fJldmd25pth851nvvPM83CMMQZCCCFuT+TsAAghhDgGJXRCCPEQlNAJIcRDUEInhBAPQQmdEEI8\nhMRZO66qauzwZ0NC/CGVtjgwmq7jLrFSnI7nLrFSnI7V1XGGhgZZXOaWLXSJROzsEOzmLrFSnI7n\nLrFSnI7lzDjdMqETQggxRQmdEEI8BCV0QgjxEJTQCSHEQ1BCJ4QQD0EJnRBCPAQldEII8RBul9B5\nQcCuA9dxu7rZ2aEQQohLcbuEfuFGJb4+nIMNuy46OxRCCHEpbpfQG5pVAIAWhdrJkRBCPF1jYyP+\n85+9DtnWjh1b8eWXuxyyLUvcLqGX1FXCK+o6IKKETgjpWk1NjfjuO8ck9O7gtMG5Oio8pgEScSH6\ne0c7OxRCeoSvj97EhRuVDt3mhOFhWJgYa3WdsrLbePnlFxAfPxpXrmQiIWEMZsx4EB9/vBVSqRSr\nV78GAPi///sXlEoFfHx8sXLlakRFDcaePV8gL+8mVq58FXl5N7FmzUps3/4ZlEoFNm5ch9u3S+Hj\n44tly1YhNjYOO3ZshZ+fP37728cBAI8/vhBvvfUOPvzwXZSWluL3v/8tJky4G88992eTONPT0/Dx\nx9sQGBiIvLw8zJnzECIjo7B3724oFAps3PgvDBgw0Ogzzz//DGJj70BGRjp4Xo0VK1Zj5MhRnT6u\nbtdC9xJzAADOPcbpIYR0QmlpCX7zm8X48stvkJ+fj0OHfsL77+/Ac8/9Gbt27UR09GC899527Nz5\nJZ566lls3foeAOCxxxahtLQEqanHsGHDWvztbyvh6+uLHTu2Ii5uGD799Cs8++xzeP31V63u/49/\nfAEDBgzAJ598aTaZa928mYO//nUlvvhiL1JSUlBcXITt2z/DnDnz8M03e8x+RqGQ45NPvsTLLy/H\nxo3rOn6QDLhdC53TnoOY4NxACOkhFibG2mxNd5XIyP4YOlSz79jYWIwdOx4cxyEmJhZlZWVoamrC\n66+vQUlJETiOg1qtKcWKRCKsXPkqfv/7RXj44fkYMyYBAJCZmYHXX38LADBu3AQ0NNSjubmp03EO\nHz4S/fr1AwBERUVhwoS7AQBDh8bi0qU0s5/51a9mAQASEu5Cc3MzGhsbERRkeWhce7hdC13GywAA\nCq86J0dCCOlqXl5eup9FIpHutUgkAs+r8dFHH+Kuu8Zj166v8eab/4ZSqdStX1JSDD8/f1RXV9nc\nj1gsBjNoJBpuxx7e3t5GcWpfcxwHnufNfobjOKuvO8LtEvqx4hMAgLqALCdHQghxtqamJoSGhgIA\nfvzxe6P333nnn9iyZRsaGupx7NhhAMDYsXfi0KGfAGhq38HBwQgICERkZH/k5NwAAGRn30BZ2W0A\ngL+/P1paumayiiNHDgIALl/OQGBgIAIDAzu9TbdL6DyVWgghrX73uyfw4YfvYcmS3xq1hDdv/hfm\nz1+IqKhoLF/+D3z44RZIpbX4wx+eQXb2dTz55G/w4YdbsGrVWgDA/fcnoqGhAYsXL8S33+7BoEFR\nAIDg4N4YPXosHn98Id577/8cGru3tw+WLPkt3n57A5Yv/4dDtskxxphDttROHZ2C7sXjq6ASNH3R\n30t8y5EhdYnQ0KBOTbfXXShOx3OXWClOx7InzueffwbPP/8ihg8f2aHtW+J2LXQx53YhE0JIt3C7\nXi5J0dPx/a2fIOb9nR0KIaQHycu7iddeW230npeXF7Zv/7Td29qyZZujwjLidgk9IXQUvr/1E3wV\nEc4OhRDSgwwdGotPPvnS2WFY5Xb1Cy+R5hzEQDdHCSHEkNsldIk2oXPm+3YSQkhP5b4JnVrohBBi\nxO0Surh1EJdmhcLJkRBCiGtxu4SuraGDc0r3eUKIi3n++Wdw48a1Ltv+ggXJqKtzj6FG3C6hi1r7\noYuDa5wcCSGkuzDGIAhUZrXF7botOmIAG0KI/f5z8wdcqrzi0G3eGTYa82PnWF2nrOw2XnrpeYwc\nOQrZ2Tfwxz8+g127voBKpUT//gOxcuWr8Pc3fh4lKek+HDp0EgBw7NhhnD79C1atWmN2++vXr4GP\njw9ycrIhlUqxYsU/8NNP+3H16hWMHDnK5HPa8dmHDRuBnJwbGDIkBn//+zr4+vp2+Dg4mtu10Akh\nPUdJSTEeeeQxbNmyDd988w3eeed9fPzxFxg+fAT27Pmi09tvbGzA1q07sXTpX7B8+cv49a9/h127\nvkZe3k3k5mabrF9UVIhHHlmAL774Bv7+AQ6bns5R3K6FDgCCLACcpH3DWxJCOmZ+7BybremuEhER\niVGjRuPUqZO4efMm/vSnpwAAarUK8fGjO739KVOm6sZX79Onj27s9SFDYlBWVoa4uGFG64eFhevG\nVp816yF8881XAB7vdByO4pYJHYwDOE1djUowhHgubTmDMYYpU6ZgxYq1Nj6hzwf2jGluOL5627HX\ned503mLTfONa+cc9Sy6MA8AgOGegSEJIN4uPH4309HSUlBQDAGQyGYqKCk3W69OnDwoK8iEIAk6c\nOObwOCoqypGVlQkAOHToJ11r3VW4ZwsdHCDiIQiA2D1PSYSQdggJCcHGjRuxZs0qqFSalvf//M+f\nEBVlPFn8H//4PJYtexG9e4dg+PARkMlkDo0jKioa//nPXmzcuA6DBw/BI48scOj2O8vtxkMHgOeO\nLgMAvDLuL4gKjnRUSF3Ck8ZwdgXuEifgPrFSnPYpK7uNZctexK5dX1tdr6vj9Kjx0A2dL7/o7BAI\nIcRluGnJReNY6QksGOacu++EEPfw6ac7cOzYEaP3pk+fgSeffKpd24mM7G+zde5sNhN6WVkZli1b\nhpqaGnAch4ULF+LJJ580WocxhvXr1yM1NRW+vr544403EB8f32VBE0KIvZ588ql2J293ZTOhi8Vi\nLF++HPHx8WhqasKjjz6KKVOmIDY2VrfOiRMnUFBQgIMHD+Ly5ctYs2YN9u51rQ73hBDi6WzW0MPC\nwnSt7cDAQMTExKCiosJonSNHjmDevHngOA4JCQloaGhAZWVl10RMCCHErHbV0EtKSnD9+nWMHTvW\n6P2KigpEROinhIuIiEBFRQXCwsIsbiskxB8Sibid4ZqydsfXVbhDjADF2RXcJVaK07GcFafdCb25\nuRlLly7FypUrERgY2OkdS6Utnd4G0Lnuj93B2V2t7EVxOp67xEpxOpbLd1tUqVRYunQpkpOTMXPm\nTJPl4eHhKC8v170uLy9HeHh4B0K1D+M737InhBBPYzOhM8awatUqxMTEYMmSJWbXSUxMxL59+8AY\nQ0ZGBoKCgqyWWzqNudb4CYQQ4gpsllwuXryIlJQU3HHHHZg7dy4A4KWXXsLt27cBAIsWLcK0adOQ\nmpqKpKQk+Pn5YcOGDV0atI+XF5TMdOAcQgjpyWwm9PHjxyM723RcYEMcx+HVV191WFC2cO79gCsh\nhHQJt8yMMjnv7BAIIcTluGVCpxo6IYSYcsuEHhLo5+wQCCHE5bhlQgdzz7AJIaQruWVmFHHUD50Q\nQtpyy4QupoROCCEm3DKhT+xzHwCAphQlhBA9t0zoj0y4BwAgkYU6ORJCCHEdbpnQvcWakouXl5MD\nIYQQF+KWCV0sFoEJHBgEZ4dCCCEuwy0TukjEAUwEBiqiE0KIlnsmdI4DGLXQCSHEkHsmdBEldEII\nacstEzoATcmFo4ROCCFa7pvQwQFUQyeEEB33TehMRCUXQggx4MYJnaOSCyGEGHDbhM4xEUAtdEII\n0XHbhA5wYBzV0AkhRMt9Ezq10AkhxIjbJnSOiQCqoRNCiI77JnSIAA4QGCV1QggB3Diha/qhAzwl\ndEIIAeDGCV3FqwFQC50QQrTcNqGLe0kBAOXNFU6OhBBCXIPbJnSupS8AoEUtd3IkhBDiGtw2ofcS\nwgEAJY23nRwJIYS4BrdN6PW+uQCAfXn7nRwJIYS4BrdN6DSOCyGEGHPbhA7GOTsCQghxKe6b0EEJ\nnRBCDLltQufrQp0dAiGEuBT3TeiVg5wdAiGEuBSbCX3FihWYNGkS5syZY3b5uXPnMG7cOMydOxdz\n587Fli1bHB6kOcyghs4YDaNLCCESWyvMnz8fixcvxiuvvGJxnfHjx2Pr1q0ODaw9GBg4qqkTQno4\nmy30CRMmIDg4uDti6TBqoRNCiB0tdHtcunQJycnJCA8PxyuvvIK4uDibnwkJ8YdEInbE7tG3XwC8\nxF4O2VZXCA0NcnYIdqE4Hc9dYqU4HctZcXY6ocfHx+PYsWMICAhAamoqnnvuORw8eNDm56TSlg7v\ns+3BqqxqgLfYu8Pb60qhoUGoqmp0dhg2UZyO5y6xUpyO1dVxWjtZdLqXS2BgIAICAgAA06ZNg1qt\nRm1tbWc3awd9zVygkgshhHQ+oVdVVelq2JmZmRAEASEhIZ0OrD0YKKETQojNkstLL72E8+fPQyqV\nYurUqXjhhRegVmsml1i0aBF+/vln7N69G2KxGL6+vti0aRM4rut7nAwOC0J568+MJrkghBDbCX3T\npk1Wly9evBiLFy92WED24kQGJRdqoRNCiPs+KWqIui0SQoiHJHS6KUoIIW6c0Pv3DdD9zEA1dEII\ncduEPj0hSvdzg8L1+6YSQkhXc9uEHhkYpvv5Xxffc2IkhBDiGtw2oYs4DtrSuZrxzg2GEEJcgNsm\ndI7T/EcIIUTDbRM6TUFHCCHG3DahU+ucEEKMuW1CJ4QQYsxtEzq10AkhxJj7JnSqoRNCiBG3TeiU\nzwkhxJjbJnQR1VwIIcSI2yb0tpS8ytkhEEKIU3lMQm9Rd3yOUkII8QQek9B5gUZcJIT0bB6T0AWa\nho4Q0sN5UEKnAboIIT2bWyd05a1Rup95aqETQno4t07ofH0/3c9UciGE9HRundAN8VRyIYT0cG6d\n0DlOPzn01zkpYDRZNCGkB3PrhG6ooKEIRY0lzg6DEEKcxs0TunGLnG6MEkJ6MvdO6DScCyGE6Lh3\nQieEEKLj1gl96IBezg6BEEJchlsn9CB/L2eHQAghLsOtE7oP/IxeU0mdENKTuXVCTxgaafT69O3z\nToqEEEKcz60T+uDIXmC8WPf6dNkFJ0ZDCCHO5dYJHWaeDM2V5jkhEEIIcT6bCX3FihWYNGkS5syZ\nY3Y5Ywyvv/46kpKSkJycjKtXrzo8SEsYADDjyvk7l7Z22/4JIcSV2Ezo8+fPx0cffWRx+YkTJ1BQ\nUICDBw/itddew5o1axwZn3U0dAshhOjYTOgTJkxAcHCwxeVHjhzBvHnzwHEcEhIS0NDQgMrKSocG\naQ1fF9pt+yKEEFcm6ewGKioqEBERoXsdERGBiooKhIWFWf1cSIg/JBKx1XWsCQ0NggocVAXxkPQr\nM1nmSlwtHksoTsdzl1gpTsdyVpydTugdJZW2dPizoaFBqKpqRE1tCyCY/gpVVY3gBR4ytRyB3gGd\nCbPTtLG6OorT8dwlVorTsbo6Tmsni073cgkPD0d5ebnudXl5OcLDwzu72U7bnLENr/yyFk3KZmeH\nQggh3aLTCT0xMRH79u0DYwwZGRkICgqyWW5xFF9vyyWbm3X5AIBaubRbYiGEEGezWXJ56aWXcP78\neUilUkydOhUvvPAC1Go1AGDRokWYNm0aUlNTkZSUBD8/P2zYsKHLg9bqHehjcx1GXWEIIT2EzYS+\nadMmq8s5jsOrr77qsIDa645BvVHstL0TQojrcO8nRUEDchFCiJb7J3QzGV0lqHU/U8mFENJTeEBC\nN83oLx5f6YRICCHEudw+oYtcvOai5FXYk70Pt5vKba9MCCGd4PYJ3WzNxYUczz+DE6Wn8a+L7zs7\nFEKIh3P7hN47wNvqcjMj7HYruVqh+T8vd24ghBCP5/YJPXnKYDDBWivduRmdo344hJBu4vYJ3cdb\nAnn6jA59Vi2o8XHWFzQpBiHEI7h9QmeMmR2gS7fcymcvV13FxcrLNCkGIcQjeEBCt748Je9Hi8t4\nxjs4GlMufs+WEOJBPCChW8/o2kG6zOnu+nZhQzFOlp7p1n0SQnoOp42H7ighQbYH6LKke9K5fi9v\npb0LAEgIHY0g78Bu2TshpOdw+xa6uSdF7VXTqHBgJOaZi05tMDQBIYQ4itsndMB2nTrrVo3Z9785\n1vW9WzpzwiGEkPbwiIQe5OdldXnGzepuioQQQpzHIxI6Z2NAl/Y+WlSnqIeCV3Y8IEIIcQKPSOhi\nWyN0tSOj8wKPVafWY/XpjZ0LqhU9KUoI6S4ekdAlIhFURXdAaO5ldrkK9t/85JkAAGhSOWZyaaqh\nE0K6i0ck9CB/L6jLY8DXRphdftHrc1ytye7mqAghpHt5REK3R0Zlpl3rObo9/XH6HgdvkRBCzPOw\nhG65WH6mLA1CazkF0Ew8IQqs072WyuvMfcyqy1VZOF+e3u7PWSvDKHklCht63rTX58vTsfr0G2hR\nyZwdCiFuy6MSurhvmcVlDAxf3fhO9/qdSx9CElGoe53fUNS6nv22XfkMn177CipeZX6fdg7Gnl9f\niL05KRCYgA8u78Rbae8iv76oHZG4v0+vfYUaeS2uVF9zdiiEuC3PSOitDV6Rf5PV1U6VnUNtg2ai\nibatYJlK1tqCb//46c3qFrPv2ztB9dsX38PxklPIkeYhp07zsFN5c0W74yCE9GyekdDb4dbtBrPv\nf5n9LXZkfeHU6TBUgr6l7+SJlrpFvaIRFS1Vzg6DEI/R4xJ6VUMzymvNt6gzqq6YLZNI5XVWR220\nxN6SS0+18tRrWHf2n84OgxCP4fajLQL6h3dUJXHwGphrdd3vsg/iP5n+8I6xtIZpEv776Q0AgGeH\nvIQxQyKQXpmJAQHmu0gab6kzCZ1OBoSQ9vGIhD50QC/cLK0HU/jaXNdrQB4EWYDF5bfKay0ue+fb\nS1j22zux48bndsVFLXRCSHfyiJLL/Kkx8JbY/6uI/Cw/Bfpezju6nwsaTHuaVDWYr8GbY6mFvvPH\nG5ApXGsI3WZlC34qOIJmlflyVEfwQtfPCEUI0fOIhO4lEeOuYaEOfyron2lb2rSyOQjtaHRbWvXK\nrRocSy82GV7AcF+dK9e032cZ3+L7Wz9jb85/HbK9tPJLWHp8BT2hSzzKwYJjSMk74OwwLPKIhA4A\nAT5eYAo/AAATHJfZ1SatTNNEu+rUerMtW2bwIFNbZxT/xSsn16K48baVLXctwwetKps1QwzXyi2X\nnNrjYNFxAMAvpWcdsj1CXEHKrQM4WHjM2WFY5DEJ/YG7oyA09oEi+y4ob0x02HZzSowTnKWkm19f\naPKetQRdK5QCAN64oC/xWEp+XVGLP158Ci8cW47y5so2S5w7mBjHcWhWtSCj0nyPI0KIZR6T0H19\nxAAAoT4MTG19wov2UKoNa90MlnJMY4txTZwXeFy4datd+7pWa1qeuFVfiOePvYLnj77S4adHzdWy\n9+amANB01TTmmCSq7XnUkdLRB5d3YnvWLmRWX3VILIT0FHYl9BMnTmDWrFlISkrCtm3bTJafO3cO\n48aNw9y5czF37lxs2bLF4YG2j+NambkNObqfJRGF+CnnjNn1PjlzBJv2pulef3njW+wp+aTjO27N\ng8eKT7a+ZPjXxfd0iw8XpeK1s2/bnJ/065wULD2+AvWKRvO76aJGsP5fwHQHJ0vP4kxZmsn7WvkN\nmqudKpn5qQMJIebZ7LbI8zzWrVuHnTt3Ijw8HAsWLEBiYiJiY2ON1hs/fjy2bt3aZYE6S6r0R93P\nXv1voaElyOxZUBJ6G3n4GsB4AMDZcssJyx7alm2JUY2d4bNre/DEyF/ju5v7AWiSXmRAuOX4S04B\nAAobijAmNN6OPetTMS/wEIvEVteultXg46tf4jfDHkFU0EA7tg98lf0fu9Zzdsnlw8ydUAs8nk94\n2qlxEGIvmy30zMxMREdHY9CgQfD29sbs2bNx5MiR7oitwxhvPQl1cutWl36XdhGCgxIRYwyVMuP5\nUM+VX8Sn177Sve7ItYi11rFWeXMllh5fgQP5h02WKXkVfsw/hDpFPVLyDqCwoRifXP3KeKXWESXb\nHorSJssDqLmaK9XXcb02x/aKhLgImwm9oqICERH6pyLDw8NRUWE6cNSlS5eQnJyMp59+Grm51p/W\n7Ap+PhIMiQzC3HuHACpfKG+O7ZL9cL7W+2kfLz6LDbsuOmRflurPxkP2ahLnmdsXjG7M5kpv4bmj\nywy2pff59a9t7jur5joA4If8gyYt5aPFJ7E//xC2X9llFK0hhVph9v0N5/9tc9/6Ldp3YlTwSsjV\ncuPPMoaUvAPIlbbvPkZ7CUzAmbI0NCjNl7QcuR9nX7EQ1+eQJ0Xj4+Nx7NgxBAQEIDU1Fc899xwO\nHjxo9TMhIf6QSDrekg4NDTJ5b/NfEwEAKb/kg6+NBHC5w9u3hBNZ7ooIAOrgQhSURsDH2jY423+Y\nQUG+2F/8k831+vYNRJCfNz4/uhcA8PWvPwAAbLlifBUVHOxn9pjVCVKj9729xQgNDUJAjf43qBfX\nQCqrR1ppJv408XGoijTJs1peg/7BoQAAkZjTbedKWa7uysLLW2J2v4YMl/cK8tP9HBDgY/azbd9b\nuOdPRr87ABTWleBg4TEcLDxm9H5HWIv/TPFFfH79a0QHD8A/H/h7uz6rVd5YiQulmZg9LBEiznwb\n6w/f/RWRQWFY/6tlZpd3lj1xugJXibM93+nuZDOhh4eHo7y8XPe6oqIC4eHGNdvAwEDdz9OmTcPa\ntWtRW1uLPn36WNyuVNrxJxJDQ4NQVdW1LaLO4Hw7Px/pRxe/Muonbom0thkqg0492uOiVhn3bKmv\nl+G2uBZX2/Sk+aXwPBYNXaB7rVLxqKpqRHOzfh7W77OO6u4JTA2fAplcCQBoUjZD0frEK68WdPte\nvTsFXgM0n1Uq1Tb/rd44pk+4DY36CS6amuQmn7X2b2/4vuETvZ39rlj7fEGlpoRUWF+K13ecxZxJ\n0RgQGmgzVkOvnNgImVqOABaE0f1Gml2nSdmM3Jr8Lvneu/rfk5aj4mSMgYFZPHnaw1ocXX08rZ0s\nbP5Go0ePRkFBAYqLi6FUKrF//34kJiYarVNVVaW7HMzMzIQgCAgJCelk2O7LOybL6nLOz/q47QDs\nSuatW7OrMFHcWIK1Z/+J7Vc+M7vc8HI+r64A12v0tWPDG7xt96WdfclSecSeskm6hekB21tgqJbV\nYt/NH6FsM+FIaVMZ9t38sR3HFGYnLSlqLMHu7P9Y7Fl07loFNn3d/qtCWWu5qMmBwy64GpWgbtfx\n70qb0j/A8pPrnB1Gl7DZQpdIJFi9ejWefvpp8DyPRx99FHFxcdi9ezcAYNGiRfj555+xe/duiMVi\n+Pr6YtOmTS4x2726PNpoViJXIe5X6rBtceDsqq0eKLDvRjZjDJvS37eyP2NpFRkANL1tpPI6hPj2\nbrNBu3ZrMZb2eP/yDlS0VCHAyx/DQvS9sLR1+8HBUejr2wf9A8IhFolxvjwd39/6GSsm/Bn+Xv5G\n23rt3L9Mtv/mhc0AgKHBgzEx4i6zMbS0XrG4Qr1bppbDT2J7wDpHEJiAqpZqhPmHmv3bf/H4SgwM\n7I8VE1/slnisuVVf4OwQuoxdNfRp06Zh2rRpRu8tWrRI9/PixYuxePFix0bWCZuen4KXtpyC6vZQ\nwEsBTqKCONh1+jRL+pbbXqkdpIr6Tm+jqkXzRGyejS87x3EW+4dvvrQNr05aBjDnnMxr5FIAsDjA\n2NmyNFypvoZ7B9yDRcPm63oLZVZfwz2R49tsy/IQCJamHAQAMOC7m/txtPgkPp1vegO4RlYLqaIe\nsb2H2Pp1OiW/vghvX9yCBwfPwJyYWV26LwDYf+sgfio8iidG/Bp3R44zu05J022z77saJa/EufJ0\njA9P6LYToqN4zJOihgJ8W89Tam+o8hIgtLjGjZSucKDgsNHwAQBQ2VKFknZ0D9yR9Tmqmu074e3O\n/g7XLAy41baLJdA1Y8JXy2psPlBlrj9nXuskJRdbryrMOV58Ch9nfWEjKsu/EwPD4aJUCExAWaNp\nb7DVZ97Av9M/0J0UKkyGXrCOMWZX6/9qay+lnwqOtmv7HaW9Urte2/093DpDYILRk9SMMaz45XV8\nlf0ffJPrmIHqupNHJnRXKPd0l3Plxl0k1YIaa8/+EzK1zMInTFmqYZuTI71peyUHtdDNJc7ShnK8\neuZNfJj5icky7V4tJVzdcARW8uHe3BRcrLS/Dm4tucqVlk86aqYGL/BYd+5tw43Z3N+rZ97EW2nv\n2hGZ/t5GV5V/btTmorb1qsj5BSbrGGM4XnIK1W2uLleffgN/O/mq7nVWzXXIec09jdtNjr2S7g4e\nmdBFIuOEIjT1trCm5/nz8ZVO3f9zR5dB3EffMm1px4mlLXNJoqhec//B/AM/mn/3w0WpZpOYfjLv\nzqUfq582WLh622mr2ylvMW6d2xNVjbwWRY0lADQ18kOFx9FipsRk2Kj58sa3Frd3vOQUPk7fY8ee\n9Q7kH8aB/MN4N2M7/nF6o9Gyqjp9LAIT0Kxq6ZITSqOyCUUNJXavf6M2F3tzUvDGhf8zel+qqIOC\nV+peN1gYIsNdeGZCb9NCF6SWH40njicK0HcZLGwo7vB2VLzK6I8NAP59+iPdz1J5HZQGyw3/1W21\nYv97qv1zxLbFGDMpaRiOly9XWp7gg7HO3zj9b94B7Mv7EV/npOjeu1yVheMlpyAyOBqny85b3Mbe\nnBT8lHu8Xfv9If8gfsg3fs5Eu7ebpfp/+4+u7MKyk2sgVdTZve3Mqqu68o01/zi9EW+mbUaLynqD\nQdVamtPOPSBr8wCaNdoTpzvxiCnobOs5JRhXdK0mG7fqC9p9c+5Q0XEcKjqO9xLfAgCU1Rj37//7\n6Q3o46unNxI5AAAecUlEQVTvHqsUrNysbGPfyXz4tY6yzHXw+5Fbl6e7PDfL6mY7P4VJWbPmSsjw\n3sW21m6pyd1wI9QSxhg4jsPl1tEy21O62HrlUwDA8JA4+El8LY4lpGr9t5bzcvh7+Zldp1HZhOW/\nrMPkyAk4XXahPb9Cl8kva0BadiUenTbUpOHpCB7ZQgeA9/4y1ei10BTspEjIe5d34EDBETQqbfe/\nN0euluPdS9uRW5dnskxbw23X9ngFIFHaXrFV2/7TNfUy5N2uh1ytMLO2Pk17DczRtRAtr2X8bp2i\n3u6We26dZliDGplpj5yOnKSUvP3HxBpb0Tcpm23+jq/8shZvpm3uVBzaXjWukswB4LVP03DgbBGy\nC9v/vbWHxyZ0Px/jiw/FtbudFAnRsmfQss9OnTR572z5RdyQ5mJPoX2Tc9vDO9b2Zb1WdnGNUez7\nzxRi/WcXzd585/rpn3sQB9fgRIn5OjoDw5Vbxr2CrtVkY9Wp9fjvLdtDPhjG03YqQwC4cMO4Pi9X\ny7Enex8qW/T7VKn1JaGixhL8JfXv+OGW9SE7zI2t37as1jZZG16L3G4qxyu/rMXzx16xuh/AvoHc\nnNXdv7PlMqW6ax6y8tiE3lZIkPnLMtJ9lCrbk0azENOHrpqVnR9KoS1xL32r9nptLvbnH7K47uYr\n7+HwpQLjN0VqNLaJy2fUL5BEXzN6z+IVBAO+TTW+4rjUOtnIyVLzY+4bKq0ynY/WMNkWlRtfDR0p\nOoETpafxYeZO3Xtnr+lvXmdWacojBwpMR9fUyq8vxE+Fpt0gv7/1s1GrvG2uMzz5mJvZy14N8kas\nO/s2rlRfM7u8XtHQ+RuwFi5sfrh1EFnV13Wvt135zK4JZ27VF5j0rOlKPaSGDog4QH75PviONW0B\nku6xfNsp+Ca0/3M/WkkyjnChQjN65dQBk8wuF/m14FDN10BrOZcT8/Abfxhf3Giznr9pScl8WcZG\nX3aDRXUWHhoT2sxWXiOvxS2jZGmcmbQ3Aw1HhVQZtBLNPUnc2KLE1fxaTBwZDpWgwtsGE6xYU1Mv\nR1a+YRnI9uTnKkGNehsPyB3LP4OKlkqjLquH0oqxYHIwbkiz8UHmTt2DVHK1AufK0i1vDMatbLWg\nhkRkPh22qFpMTnSZ1VeRWX0VfxqzBKP6jbC4/X9d1Dx1PTZ0VOu8BV7oyk6eHp3Qf5d0B744pO3e\nxoFujjqXV0zb6e5ci7WSUItY38ryijL/YJU5aZUZmBMzExKRBEHe+kHs1p39J3xHme+hYXijddWp\n9Xbtp1pWa/SEbNsYO3ILdvO3mcgrbQA4YFSs5YfzGGNGT9a+9mkaWhRq3U3nbRbGDxKYoBsga/Ol\nbTYfyTf3O5xo2QNlWg28emuuhH4pPYc5MbPwbe73uhO1PdvbeP4dKHgl7mnzlCtjzOxVidYHmTt1\nN+3bMrz3crkqC5ersgCv6fC78xgyGuQYi9lW4+sIj07oUeH6PyCxiJK5sxmWOVzRO+mdG2bXHLWg\nxt9PbwAATIqcoHvfVv98XuCx9qz5RGHOuxnbEeQVaHtFcGhRybA3NwUNMsvDV5c2lSG/8RaAfqiq\nkwOc/U9ba8ezsYUxhhOlZ3C5KstiMq9sqUZvn2B4i73MllM4LxXONR/Avb3vMXq/2I4uh4YJV/tM\nwMk2E7WXNN3GkaITNrdldvtmTkDiXpp7GOfrUvFkFyR0j66hxw3srRsG4PlHR7v+42zEqcwNXeBI\nZ9rR26JGLtWNTWPO/tJ9Ju81qiz3IjJsjW6+tBXny9NxQ2E+Hqm8DhvO/xs+w9Pge9dhqJj5spGW\nzOCKgvOyvu7xDP14LoWNxdiT8x1uSC0PF7D27FvY1FrqaddVhh1dAl88vsr0Y21SYmdq8mZHl7Qx\nn0JneXQLHQDefXEqBIFBJOLAlHRjlLgL64nkan37yleprb1tOADFNgbJ0l5RAAAnUaNWuA0Ogyyu\nb9jLRRxcA+/h5yHU9zO/bnkjvFvHJdPWl23RxmtPcm1UNUFgAoobbY9oau4E0SRTOywrmkvo3kOu\nOmbjFnh0C11LPxQAB3V5NABAkAWAGYw5wgTLZ3RV0bCuDI8QExlV1sfU7yh7yyGG2lusFPeqhdcg\nC3Oximz3dLLEagvdYFF7xiZqi29zeOzpCy+VGz8JK5Vpbu4yJ4z/7vEt9LY0Q+oqoS4dCp/hFwBv\nBQSZPxRZ98JnzEmIfExrm+ryIRCHF0LkY/9jw4R0RkregS7ZLmPtT9DpqgOoz7xhe0U72DP9ojm2\n5lQtrdPfnzla3PGebCLf9o899PfTG/Be4lu4VHkFGRVXkVaVjgE+gzF/RFKH4+ioHtFC1xoRHdI6\npO5YMHkgAv28AQBCUwjARGAKfUmGGc7rBkBx5d5ujZUQV5JX3/mxbwAA4vZfIQCam8vWTgX5Lfor\ngs6MH9QZH2XtQlqVpmdNqaIA72Zs7/YYelRCf/nXCXj3xft0r/VjKWi+KrpyTFMwlLdGG39Y6HEX\nM8QDcRL7x7vpCl4DTIdvsAfPeJeYBcqS1W1GnXSWHpWlRCIOAb76lrduvIvW/wl14ZBdmAkwEUTB\nVSafZ4IInB13qQWFL5VnCHGgazXZ+Pbaj84OwyJrPZK6U49qoWv9LukOhIX44eGhDwAAHh11P0bF\n9MGzD8cDTHNI5t4brVv/iQc0N0Xl6YmQpU/vkpjalngIIXofX/3S2SG4hR7VQteaMW4gZowbCAC4\nu/+dEHEizBypWebnI0ZNvRx9IuqB1kb6/QkD0CJX45vjeYAggbpiECTh+jqd7MJMeMVcgaSv5cGE\nGC8GJ7Z8h191eyi8ox1z44kQ0jP1yBa6Ie2jx1pjhvbD9LsGIthH82Sc9nHth+6JxqonNI8F83Vh\nxhthIpOujW1fK7ONJyFuizX3anfshBBiqMcndEsG94rCU6MWY/mEP+ve8/VqfVTaoP+6uiIKMycM\nAlS+UORqRp7iqwdAXT4ETK25AFIWjLS6L0VuAoSmPlDXRJgsU+bHQ3FtYmd/HUJID9AjSy72uits\njNn3hcY+4Ov6ga+JBF/TH79ZFIewED98fhCQZ0zVPZEqv3IvxEG14GsjASv9bwWpJpHzNf0h6auf\n3UVZOBx81SBw3qZzRsouzoDfONMR8gghPRe10NshvI+/5gcmgjJnPPiaAdB2kUm8ayD+/cK9uDtu\niO49qHzB1/bXvGYiKPPj7d6XsmAk+IrBmt2Z6zLJO+Ymqqp0KJhAXwNCPAH9JbeDRCzCi4+NxfQ7\nB5hdHhzgjadnj4S3l4XDyqw/oyc0a6bJU1dEga+M0i9Qe9sVn7oiCvKrkyDI7R+zRl0aB3nazA5N\n0cfXmR+vgxDiHJTQ22nM0L54fJblsV1EIg53jwi3e3uMNxjCVOUD2YWZUBVar7m3pchNgDzzXqgK\nR4A1B0No6NuuzwMA39hHH5PadiWOMUB5MwGKG9Zv9hJCug8l9E7S9lE3dN/Y/kavn0nWJOi2iVZ2\nMRHy9ETjDzPz/yR8vWmSZkofzXal4WDyQOhKPQb1er6+L5jSnha+/jPy9BlQFcfpSkRt+8gr80dC\nnpYECBIIDa7XSnfkA4XmjjshropuinbQBy9Pg0TMQSwyTcCxA4zLF2Nj+2HYoN7ILgYGlS3E43MH\n4B9bMwDevlIKoOn2GB/TB1xICa62jsApvzwVkKhhbbglZe5d8BpyBZK+5RBkARD5WZif06gcxEFd\nNhQAIKsaBHG/EnjHGIz+x8Sa/2yQnZ8Jv4n6SYflV++Bb7zxBAKK6xPAeSnhHXvZ7DbU5dHgfFsg\n7m365K4l8gsPwHfcQXDizo92x+T+QHD3zQlJSGdQC72DfLzEZpO51oB+AQCAfy69D34+EkT01dxQ\nDQsJwIDASEDlo1t3zuRos9vQmnfvELzw6Bi8vPBOLE2cjb/MnayZgYmJjbYDAJPiI6Cu0GxPkZsA\nCJYTrzJfX9pRlw+xuB5rsw/75xkQ6a4iAIA19zZaqrh2N4TGvhCa9O+rqzT3J/iGPlDeHAtV0Qgo\nc/TTggnNvaC4MR68NBSyC5ZHs1Ncv9veIK1iCn+6V0DcBrXQu8iKxXehtLoZw6P7oKqqEQunxyKi\njz/uG6Mpx/z7hXux/rM0PPtwPAaGBuKH06azoffy98Jff3MnBobppxaTiEUYNaQvxCIOvMFEwTMn\nDEKLQo0/PDQCT6qGobx2Btac18xIwxSak4nQEgSIeIh85FDmx4OvMpi0QO0NRc6dZseqEer7QVU0\nzGCeSuMrAnnGNHD+jfC5QzPSnOzS/eBax72WZ94Hv/GHwUtDTbarHY+eKf2gLBgBoak3mMIfTOEH\ndUWU+SsYxkFo6AelmVKPqugOqGs0JwTWEgx55r3wHfOL6TZsEFoCIfJvAuNFmpNjRTT8Jhy0+Tle\nGgp1RTR8hqe1e5+EOAIl9C7i7+uFuIH6lqefjwSzJup7rgQHeOOtP00GAKjU+iEB1v/P3QgO8EZt\ng8IokbelndD43jGR+NW4gYgK18/56O0lNnqtvj0UTOkDvqa/prQiEvDguFgcqCoy3madpZu5HNTl\nQyxOjsyUfmBKP8guztC8wXvpG/GCBLLzs9D2JCAo/MBa9DHylfqrFPXtWMu/tzzA6LUi50743HFJ\n09pvCjGOSx4IQe4PkW8L+Lp+EGRBEBpDdCcecxS5CRDq+0HcrxR89QCz9zQEhS+U2ePB5P66kpJm\n/71Nfs+21BVRkIQXmV3GBBGExhDwNZHGJS4HU5UNhldkQbs/JzQHQRTQ6PiAeiBB7t8l26WSiwvg\nDOY/jOwbAH9fL6vJHNDf+AsO8DZK3mYJYvCV0Xhg3FBAkOCBcbFYcP9QvLRwrG6Vxx8cgZd/k2D0\nMT8f4/O9ujpSs7lmC10ceS8L/eP1v588axIUuQlQXJ5mVId/5D7LJR8AkKX9CqqiYVAVjjD+1erC\nITv/gEky16/Q+hUXxFAXD4NQFwa+VnPiUuQmgK/rB2XBCE1M1ydoHvISJJoTjJn+/0ztBcXlaa03\noUVQ5NwJdU2EXckcAFSFI6G4pikHCc1BEGT6E5Q8434osyeA8fr9amO1lzxjGlTFcVZn2TIc99/u\n7V6ZAsXVKRaXCwrfdm/TkGFpricQumioD0roLsCO+WxNTB6lebrU8CrAloWJsfh4eSIWTo8Fx3EY\nFdMXHy2bjm1/ux8Lf3UH4gf3wdJHNU/HDgoLNEpPU0ZFYP3MP0KenojJsUPxm8RYLHlwOOZPjdGt\nM2ZoXwT4SvDY/UMtxsBagiFII/DKb+/EysfHYcroCHzw8jTcO6a/yboPGFzRvLZkMj588g92PVA1\nPKo3xg/TlHhUpZrWvrpCvy3lzQTI0qdDkEZoHhCrjIYgjYDQaLlHi7JgJISmXpBfmg7DxC3UhUOV\nlwBLyVx12/REJTSFQHb+ASiuToHiin58fu3zBkKdvjylvKk/ySoLh0OQ++mSddvnDWRpvwJT+kFd\nNtTkngjfEAKhOQjq8mjw1QMt/p6WMKUmYStvjTI7RIXi8lSb21BcnwB15UDI0hNNYpdfnQxV0TBd\njyq+0f7vNQDIL99n9JqXhkFVNgS8NMzs+upq0++bMk//ZLj2Xg5gPC6TuW666ur+JlNY2myBq+zv\nENEedpVcTpw4gfXr10MQBDz22GN45plnjJYzxrB+/XqkpqbC19cXb7zxBuLj7X8qsqcTi0RY9Ks4\n9O8XYHvlVk88MAwzJ0ahf1/LX5z/nTcKvMCw9b+WJ6YViTiIDJJRQlw/bF92P8QiEV545wQAYPpd\nA/D4TM2XesdfHzDZxoP3RCH/diNi+vfSzd+697hmIoM74/rhUm61yWeGRWla1NoeQT5eYjwyNQZx\nA4IhEYvQp5cP+vTyxb1jIqFQ8RgQav6K5a0/TcLF7CpU1clwNL0UAb4SLPvtXVCpeaRlp0KQRhiV\nfFYsvgsbP08H1Pa3CGMHBONmqf5hr769fDFhRBh+OqcpnWh6MGnmldzxynScLgrBl3m7oa6OhLpk\nGESVcYi7g8O1Ww1mty+7OMN4Nngm1ozLDwaAg7JwOLyjb0CQhkHR+vSwuiJaUz4TqyGJvAV1xWCL\nk7DIr0wBkxl0a23dpySsGJy3HHxNBLwGX4PIvwkAoMgeB59hFzXrnZ+lmWWo9UTKVw8EXz0QQn0J\nmDwAkgE3ob4dA1ttQ+WtUZob4K0nTdWtMfAZeU6/gsoH6vIhRicir6jrEOQB4CujwPm0gPNvBCdR\nAExkUpJiigDdKKiq4jhdLy0AEPWuNCqzya9MBpMFQdyrGpy3UvPe1XvAmnuDDy2BuFctmNIXimt3\na+ZAMJiW0vAGvi70/FEAL9GV0hTXJkJoCoHfxJ81n2mdH4GXhkEcUgkmiKAqHm71eHWUzYTO8zzW\nrVuHnTt3Ijw8HAsWLEBiYiJiY/V1zhMnTqCgoAAHDx7E5cuXsWbNGuzdu7dLAvZUSeMtz6pujlgk\n0vWksWT8cE3rRM0LCA+xv2an7b3zxAPD8cG+LItPxhquHzvQuAzzv/NGobiyCY9MjYGaF/DpgRs4\nlVUOsYjD5j/fZ3Y7yZMHm7zX9iS3ZE48jqUVISzEDwlx/dAv2A+zJkZBYAxTRkciurX85CUR44lZ\nw/DZz9n4nznxqKyToaCsAUP7m5aL+gX7orpejnHDQjFldCQCfCU4ml6Kc9cqAAArHx8HaaMCtY1y\nDInQn7QWTo+FmhfQ0KzEX98/DYlYBI7jMCX6TuzYrZnw4NmH43H3SE3Z5Fh6CXJK6sELDGk3KgEA\nT80egf79ArDzxxsoqWrSB2VQu+crBmNa9H04LSlHg1Iz4xAHkeY+Be8FdYnmZDsgNAClVabdUpnM\nTEmO94K6TH91pSoaDu9haVBcmwTW3Au8NAx8Q18AnC6ZP/nAMHz6k+Y+iraVr8yeoNuG9mayIvdO\nQOUFSWQ+lHljwUlUuvGNdOs2hUCeMQ2+CalmW/yamPTlNc3Ncu13mEGh9IPQ2Bt+Ew7pnnJWFcZD\nVTzM5MQm1IUa3dTXHA8O8gztMyACtCckZd4YSEJLoS4frOshxlp6QVU6VNOpQJBAfvUeAADnpYDQ\n2Ecz2mrhCKjLBoMpTf/OFFmTwfnKwJp7Qdy3DHxDH4vPm3QWx2zM63Tp0iVs2bIFO3bsAABs3boV\nAPDss8/q1lm9ejUmTpyIOXPmAABmzZqFXbt2ISzM/OUOAFRVdfzmSmhoUKc+353cJVZLcTLGjGr8\nHdUkU+HA2ULMnBiF4ICOX26293gqVDx8vIy7bmbl1+DyzRqcu1aBJpkK7780Fb7epm2bmyX1EBjD\nHYNsX/5X18vQO9AHErHmDzW/rAHSFhXuGmpaxpEp1Pglswz3jonU3adokauQmVeDCSPCwBjACwzV\ndTIE+HnhWkEtJsVHGP07MMbw/r4sRIUHYXJ8BBgY+gX74dCFYuw+kovfPzgcY4cFoa5FjrXbrgAA\nPl6eaPT5jZ+n42ZpvUl80RFBKCzXH+MlDw3HpPgISMQi/OGNowCAt/93Mr4+dhNz7x0Cb4kYf/vg\nNMAJWDx7ED7/oRQAsOXF++AlEePZt4+bPWYTR4ThfE4pwEtwT3wEHp85DAfOFeGH0wW6dbQnqZkT\nBoEHcM+IMKz/7KJuefKUKHx/qhAAhymjIiAwhjNXK8z/I4nUiIr0xe+TEpB2owo/ntX3LBsSGYT5\nU4fiakEtwICfzpu/ca31yNQYHL1YgtiBwbiYrXlG4n/njcL7+/RXDuJ+JYCYB18RjWeSR2Lb99d0\ny+IGBmPF4nEm27VHaKiVe2bMhgMHDrCVK1fqXn/33Xds7dq1Rus888wz7MKFC7rXTzzxBMvMzLS6\nXZVKbWvXhHQ5QRCcHYLDyRQqo9cnM0pYYVm91c8oVTxLu17O8m9r1hMEgTXLlEzZ5u/0x9P5LOXE\nTavbKqtuYpm5VbrXReUN7P1vMphMoWLvfn2JfXc8lylVvMXPN7Yo2bJ3T7D3v8lgal5gReUNRsuL\nKxpYY4tS97pZpmTZhbUmv8/tqiYmV6pZTb2MZeVVs6NpRSZxllY1sqLyBqZQGv+ezTIlyynSb5Pn\nBXartI79fLaAFZU3GH1vyqqbGM9rXrfIVayovIHV1suYIAjs26M57EZBDWOMsYraZrbnUDb7+WwB\nq65rsXoMO8pp3RalUtMhYe3lLq1ewH1ipTgdz5mxGu51WH9NjwpLsYSGBqFO2oyo1vsx1mIeH9vX\n5jpiABHBPrp1fEXAgqkxaKyX4detN8zrpBaeWG71119rbgbX1jTBV6TZn/Z4+nCArEkOWZN+3t4Q\nP4lJTBIADXWaPBMW5I2wIG+jdcSt/3mJgPo603zU29d4m4FeItwZoxnzqLpaXx4TA6ip0b8eFK6J\ns1qhwn2tnReqqhrBAZg+trWnmFLd4e+GtRa6zUJOeHg4ysv1Y3RXVFQgPDzc6jrl5eUm6xBCCOla\nNhP66NGjUVBQgOLiYiiVSuzfvx+JicYDSiUmJmLfvn1gjCEjIwNBQUFW6+eEEEIcz2bJRSKRYPXq\n1Xj66afB8zweffRRxMXFYffu3QCARYsWYdq0aUhNTUVSUhL8/PywYcOGLg+cEEKIMZu9XLoK9XJx\nLRSn47lLrBSnY3V1nJ2qoRNCCHEPlNAJIcRDUEInhBAPQQmdEEI8hNNuihJCCHEsaqETQoiHoIRO\nCCEeghI6IYR4CErohBDiISihE0KIh6CETgghHoISOiGEeAi3S+gnTpzArFmzkJSUhG3btjk7HCQm\nJiI5ORlz587F/PnzAQB1dXVYsmQJZs6ciSVLlqC+Xj/N19atW5GUlIRZs2bh5MmTXRbXihUrMGnS\nJN20gB2NKysrC8nJyUhKSsLrr7+OrnhswVys7777Lu677z7MnTsXc+fORWpqqtNjLSsrw+OPP46H\nHnoIs2fPxqeffgrA9Y6rpThd7ZgqFAosWLAADz/8MGbPno3NmzcDcL3jaSlOVzueAGxPQedK1Go1\nmzFjBisqKmIKhYIlJyez3Nxcp8Y0ffp0VlNTY/Tem2++ybZu3coYY2zr1q3srbfeYowxlpuby5KT\nk5lCoWBFRUVsxowZTK3umqn4zp8/z7Kystjs2bM7Fdejjz7KLl26xARBYE899RQ7fvx4t8S6efNm\n9tFHH5ms68xYKyoqWFZWFmOMscbGRjZz5kyWm5vrcsfVUpyudkwFQWBNTU2MMcaUSiVbsGABu3Tp\nkssdT0txutrxZIx10dTTXSQzMxPR0dEYNGgQvL29MXv2bBw5csTZYZk4cuQI5s2bBwCYN28eDh8+\nrHt/9uzZ8Pb2xqBBgxAdHY3MzMwuiWHChAkIDjae4b69cVVWVqKpqQkJCQngOA7z5s3rkuNtLlZL\nnBlrWFgY4uPjAQCBgYGIiYlBRUWFyx1XS3Fa4qw4OY5DQEAAAECtVkOtVoPjOJc7npbitMSZ31G3\nSugVFRWIiIjQvQ4PD7f6Re0uS5Yswfz587Fnzx4AQE1NjW7GptDQUNTU1ABwfvztjavt+xEREd0a\n7+eff47k5GSsWLFCd9ntKrGWlJTg+vXrGDt2rEsfV8M4Adc7pjzPY+7cuZg8eTImT57sssfTXJyA\n6x1Pt0rormj37t1ISUnB9u3b8cUXX+DChQtGyzmOs3o2dxZXjUtr0aJFOHz4MFJSUhAWFoY33njD\n2SHpNDc3Y+nSpVi5ciUCAwONlrnScW0bpyseU7FYjJSUFKSmpiIzMxM5OTlGy13leJqL0xWPp1sl\ndHsmrO5u2v337dsXSUlJyMzMRN++fVFZWQkAqKysRJ8+fXTrOjP+9sblzMm/+/XrB7FYDJFIhMce\newxXrlxxiVhVKhWWLl2K5ORkzJw5E4BrHldzcbrqMQWAXr164e6778bJkydd8niai9MVj6dbJXR7\nJqzuTi0tLWhqatL9fOrUKcTFxekmzQaAffv2YcaMGQA0PWL2798PpVKJ4uJiFBQUYMyYMd0Wb3vj\nCgsLQ2BgIDIyMsAYM/pMV9P+QQPA4cOHERcX5/RYGWNYtWoVYmJisGTJEt37rnZcLcXpase0trYW\nDQ0NAAC5XI7Tp08jJibG5Y6npThd7XgCdkwS7UosTVjtLDU1NXjuuecAaGpsc+bMwdSpUzF69Gi8\n+OKL+Oabb9C/f3+88847AIC4uDg8+OCDeOihhyAWi7F69WqIxeIuie2ll17C+fPnIZVKMXXqVLzw\nwgt45pln2h3Xq6++ihUrVkAul2Pq1KmYOnVqt8R6/vx53LhxAwAwYMAArFu3zumxXrx4ESkpKbjj\njjswd+5cXeyudlwtxfnDDz+41DGtrKzE8uXLwfM8GGN44IEHMH36dCQkJLjU8bQU59/+9jeXOp4A\njYdOCCEew61KLoQQQiyjhE4IIR6CEjohhHgISuiEEOIhKKETQoiHoIROCCEeghI6IYR4iP8H9L/g\n+GJ1H98AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbe2541be48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sns.set()\n",
    "plot1, = plt.plot(maxout_mlp_loss, label='maxout_mlp')\n",
    "plot2, = plt.plot(relu_mlp_loss, label='relu_mlp')\n",
    "plt.legend(handles=[plot1, plot2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7fbe25304e10>"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAIABJREFUeJzt3XlgVNXZ+PHvnS37vpKFJQs7BBDEgCIECChbimhbW6vR\nitoqUq3+WKpVq6Va21KrrcSl4tK3tGjD+0pfi0TE9RUUEFBZFSEQkhASSEK2mbm/PyaZZDIzWSeZ\nmcvz+Sczdzn3mTuZZ86ce+45iqqqKkIIITRF5+0AhBBCeJ4kdyGE0CBJ7kIIoUGS3IUQQoMkuQsh\nhAYZvB0AQHl5da/2j4oKprLygoei6TsSp2f5S5zgP7FKnJ7Xl7HGxYW5XaeJmrvBoPd2CF0icXqW\nv8QJ/hOrxOl53opVE8ldCCGEI0nuQgihQZLchRBCgyS5CyGEBklyF0IIDeo0ua9cuZLs7Gzmz59v\nX1ZVVUV+fj65ubnk5+dz7tw5+7p169Yxe/Zs5syZw/vvv983UQshhOhQp8l98eLFPP/88w7LCgoK\nyM7OZsuWLWRnZ1NQUADAkSNH2Lx5M5s3b+b555/n4YcfxmKx9E3kQggh3Or0JqZJkyZRXFzssKyo\nqIhXXnkFgLy8PG644Qbuu+8+ioqKmDdvHiaTidTUVAYNGsTevXsZP35830TfB2rqmvjfT75lzqSB\nhIeYurzfF8fOcrriAjMvSXFad+ZcHe/uPsWi6Rk89tcdXDI0jkazFX3C14xOzMBaHcUr/znId6al\nMWFoHP/3xWl0OoXa0MMEq1Gc+DqQRZcP4bNDZegUhUtHJFBnruM/x7YxPXUqeksQb+04zqjB0RR+\n8A2XDo9n1sRUACrO1/HCp5u4LHk8x47B3qPlkHiIiMY0as0XCIk5z1VZI9l3ooS6U0nsOXKGuKhg\nRqYF8RGvMk6Zz6TUkTy1cS8Aep1CVNpJrhg5hIPlx0mxjsPcpCMqLICjJ8+z50g5huQjpMZEcDpw\nF4GWKK5MmEnhv2vQB9ZhGrsdTmdQdzwDgIEDFQZknOWTbRGg2uoaM8YnMzAhlK9qd/Pp5xewno8F\nYNLweIYMCCcxtY7nDrzAovhb+PubJwkJNFBbbwZDA4bEY5hL0sBiRK9TWDBlMIUffGN/L37+vXE8\n+ffdGJKPMCAshrKaczQUD3F4v3SRZSiGRixnUhgxOIzD5k8JqkmjusrIwssHsaXmJRRTA02n0pga\neyWTRyTy203vYkw9hPVcLObTtvKUwBoGjaqkTPcVTRUJBBJOg6kcffhZmk6mYz6ZaTugYsWQfASs\nOhRTPfrYkzQeGU/A0F1Ya8JpOpmJKXMXjUfGo48sJyk6nDLDF6hNJuo/n4Yu4gyGmBL00aUANH49\nmvSBQQxPSGVL6X9jqYrH2BhFo1qPMfkoDYcmYEr6BiW0kqZTaRiTvnZ4/db6YHSBtptuVIseRW9B\nVaHxq8kEjPwEc3kSI+IHcVj9GOuFUNSmACxnkjGl2/5HGg6Pw5DwLfrwSgIvpFIffAKAphNDMaYe\nQrUqWM4mohiaaDoxDLUuBGPafgyxp2g4NAFrbQSGhGPoQqtQDE0oAXVYKgagjylB0dsqiw2HJqCP\nLAfFiqJvQh9dZou3yYilIglrQxCG2FOgM9N4+BLU+hDb/2/8t5gGf2V/rapFj6UyHvOpDIwDD6CP\nLMdSGY8upApFbwW9mYYDkzAO/ApdcI3TZ7vh8DhMQ75AMTShNplQGwNt53fIfvSRZ7CeHI653oja\nGIgurBL0ZowDjtFw8BIChn2GtT6I2RHfZ/HU4a7SSq8oXRnPvbi4mNtvv50333wTgIkTJ/Lpp58C\noKoqkyZN4tNPP+WRRx4hKyuLRYsWAbBq1SqmTZvG3LlzOyzfbLb4zE0Ja/++i6KdJ5gydgArb7y0\ny/stuHcTAIW/XYhepzisW/a7bXxz6rzjDsZ6gsa/C0Ddjtbz8z+/W2QrS2chaOLb9vW35o3mucL9\n9m1e/GwDbx15l6zEkQSfmso7n55wKH7Tbxei0ykse+7vnA7fjqpC/c656KNLMGV8bv/QttU2jsDx\nRSjGJqflGBoImrDN/rTp1BDMxcPsz3VRpwnI3ON0fup2zCXo0recjhU4bhuKqYHGb0ZiKR/YZg+V\noEv/43x8cFkOgCnzM/RR5ZhLU2n6dpRTDPYYI8oJGPaZ/Xn93stR60Odyq/bMRdD4jcYBx7EWhdC\nw74r0EefwpSx175tw1eTsFbHOMa0awaYAwicsBXFYHYbR0vs+rgTmIZ84Xa7jjSdTMeYfLRH+/qK\nxqNj7V8MAJbKOPRR5R4rX20yUb87h7b/U/1NtSooOtep1lyezBt3/sLjx+z18AOKoqAoSucbdqC3\nt+bGxYX1egiDFqfP1AJQeqa2R2WWl59Hr3Ns7Tpd4fz6FMXqZv+WYzr+I5SU1ThsU1Jl++cvrz5L\n4BnnGkVZcxxna6shHOxvkd6WsNsndqf4mhO703Kdtd12jY7P9e6TmcvyTA22v4Z2x1O6P4eMElDn\nMiYnunav3c17AYChsbns5vewfbJ2dR6bY+8osTts3v61d4Ni6OS1+oN2r6Hlf8JTOv1/6AfuEjuA\nYqrvcf7y+PADMTExlJXZfgaVlZURHR0NQEJCAqdPn7ZvV1paSkJCQk8O4bdc/Q7q3VefjdXqWLBF\ntSUkvaJz+eXab/Nr9SAJuy6nj8p1RXU6WBd2creNqzg98Y53UV+eJ625yM5Vj5J7Tk4OhYWFABQW\nFjJz5kz78s2bN9PY2MiJEyc4duwYY8eO9Vy0vVDTVMuJ6pMOyxqaLBwpPkfblqkzVbba39FT53j3\n4D5KKts1p7Tdv9HCkZPnOHOuzr5MVaHJbGXv0QqOnrL1IrrQ4FyDU9q03ymBNWCsB6CyurnW0qZ2\nqYRU8Z9PW9tF3959lENnbc/P1J7nbHUDushSjOmfg6EBUHl263s89fpuai6YHcrpyU94XdhZ29+I\ncvtj+7rg82BoQBdWgRJU7VwrbnPstvTRJehCK1ufx5zCOGQfuqjTKIE16GNb3ysl+BymzF3oQivR\nJxxzLCf+OMbB+9HHH7e3ieqjSzEMOArGevTx34LOjC6sAn3CMQypBzCmHHYowzjoS/QJx9CFVqIL\nr2gtO7YYXbCtRqXorC5r+MbkoxhSDjksM8SetL0XnTCkHAJ9E7rInjdBGOKLO9/Ix+mjSh2e60Lc\nf+Z6KmDM+wSM/tDj5XqCYqrvm3I7a3O/55572LFjB5WVlcTExHDXXXcxa9Ysli9fTklJCUlJSaxd\nu5bIyEgA/vKXv/D666+j1+tZtWoVV155ZadB9LZJpSvNMve/9xC15gs8fsUvCTXaLq48tXEve46c\n4WfXZTEmLQaAm3/zDmC7qBYwdBeWswk8u+Rel2X+fsMe9n/jmOyevfdK/v7OEd7dbUtOty8axbOb\nnNtT27bRtnBoPx7xf+jDWhOitSachi+nABA4ocjhp3z79uzGb0ZhGvIF5tKBWGsiHdozO2I/vqGR\noAnvOKxrODzOZVv6xcRcnoy1NgLT4C+9HYrQmGdynujRfh01y3Ta5v773//e5fL169e7XH7HHXdw\nxx13dDG0/lNrtrWZXmi6YE/ue46cAaC4rMae3Fvogm21h5YeCK60T+xg+5G+53B5h9t0RdvEDqAL\nba3NdNZG21Ij1kWWYa2J7PaxFb1z+fqIM90uR2v0Maew1kZ4OwwhukTuUAWPNpGqqurQBq5avdDO\n1+PX0xJrP7YZ+xU5L8J/XHTJXXV6pqKgYLG67z1isdraWq2qitVqRW3+21qaan9stapYVat9mdmq\ntlnfdvvOo+v5Nj3dvm2MwjU5P8I/dKmfe1/rjzb3n75zPwDjzEu4NfdStuw8QWFFAdb6YPQYILyc\nOSmzKXxDj3HwfocLVS1t0S19uNv2w23bX9xyNgFLZTym9H32fdv32W0qGYy5ZIhDX/G2mk5kYi5J\nd9kmL4TQpr5oc7/oau7/96WtDf3vRYdRTA3owysh3JZ8/1Nsu2nIXQ8EY6qtV0TbZN22v7g+uhTj\noAMO+7S/GcM44JjtTjU3jKmH3a4TQoiuuuiSe5/rwu8g5SLrbyuE6H+S3IUQQoN6PfyAv9FHnebl\nvYUoga6/1wJGfuy0zDRsJ2pTgH0wpY64u23fobyMjm9wkfZ2IURvXXTJ3Zh6mE/OHMY0NMjlel3o\nOadl+ogKF1sKIYTvumibZXozWJMQwnsuD1rs7RB4esbj3g6hUxdtchdC+KfejkKrlRg6o9lmmdNn\nL3D2fD1x8Sonz5c5re/qcKxCCN/iB3nVJ2g2ua8q+D9ALk4KoTUZ8Ym8903n213sNJvchRCdyzSN\n53Dj7j4rf2zwVK7NmsYDH6/pcLslmQvZePi/u1TmJUMGUWv8LhsObfBEiADMHzKXUTFD+dWGd9FH\nnbZP29cVsxLnYrAGMiw5AZ2io85cR1xQLL/65Emnbe/M+jEnqk+y6ev/9Vjs7kibuxAXsUhjdJ+W\nPz1pGtFBUQQZAp3WTU68xP44PWJwt8qdmjGy27EMDrdN45gY4jyB0CUJYxgYnoKlIgnVaqvzGnRd\nq/tOT5vAgtFTGBqVTkbkEMbEjiQxJN7ltiNihpI7eEa3Y+8JbSd3nbSrC9GhfmrAdjlDWS+O3ZN7\nvJUOR/VsXac0Tzhj0hm7Vq6PXgTQZHJ//s0vbRNQT9zq7VCE8LjIAM+NKR8eEOKxslwJNLXUfp3T\ncZixdVJyk75ribSFqnYw760bsUG2XymxgVFO64xtaulqUwCA29q3v9Bkm/tH+0+ji/DMhNnCs2ak\nXs62Ex+QYB1Bqe4rb4fTJ0KVaBYMnY5e0fPqgX92uO3CtLkU15xiV5nr2bKuHjwLgH8fa62o3D72\nJv73WBH1jU00VYfwtXWX034rJi3ns1Nf8PbJt53WJYYkMCJsFI0VcSwcM46idzc5rL997E28uP81\n7hp/KxV1lZTXneFc/QW+Ov0tFarzoHo3jvwe67/8u8Oy0ZFZRDcMZ2CCLYGrzck9OjCKs/WVTEoY\nz6WJE3j7+Lv2mNoy6AzMTJ3GwPAUntv3MgCzBl7JqJhhAFjbfFmYTw/CkPit/XlW7Cg+P9M6+9mg\n8FQuTZzA5MRLiA2KYVpKNl9UHCQyIJwgQyDfnDtOVGDrpDZNxZnMHp/GlJQJPLajdbKiZeOWUlZ3\nhsRgx6SvuKkj/2D4tbx24J8sHjkXS73CqNjh9nXLx99GSW0pm795m2syF7jcv7c0mdwBF5MgC1+w\nJHMhSzIXAq3DMGvN4zNW2B+3Te7JoQM4WVPisO2cwTkcrTrmMrkvSJvL3ME5AJyqPc2e8v0MCEkg\nNSyZpWN+1GbL7/HWsXf4n69tPcNaho81puidkruroWVHxgzjy4qD9ucRAeH8YfpjAKS1bQsf4fye\nZcWN5tLECQ7J/U8zfoNOcUx4Lal4XNxoezIru+B+7tg/Tv+107KpSZcSHxznWCBgPj3YIblfkpDl\nkNzTIgYxPWUqAPPTcgHIHjDRvr6lLd7OYmR2ynRUo+PcpilhSQyLznCKy12rzJSkSUxJmuRySPLM\nqHQyo9KZljLF9c4eoMlmGRtJ7sK36Lr9P9mawVqmXXDXbuy67K4dr32ZHbdNt+OiMb2j/btVttO+\nrenK2mGzTC9ej30nxT5JT+dx+Wau0VTNvbqxhhf3v4YSEicT5gjf46aKp3bhn9U+AaK7aqKLxV1N\nOc7b9S5ZuYyx5UugF0V3/bql4/nsSfJVcP4CcVeKryZ3TdXctx7fzqGqowQM+1SaZdoIsHR/kuz+\nkBKa1Ok2CS0/w4Hx8WOZPXA6lvNRmCsSu328kdHDSKxv7X4XrsS63TY2sOtdBK0XQjvfCLh++DVk\nxY12Wu5qMrRAfQBTkya3btPJF8CUAZeSHDqAn2TdYl8WFxxLZmQasfVjO9z3OxnzSQpJZFpyNsNj\n00ly0VWwxdVDZjMuboz9+cL0qxzWj4l13UWxdXZe95/L28feBMCitKtcro9ucyHU2uacZSYkEmdI\nIcQQzPj4sYyOHcnAsBSmJWeTHDqAnIFXuD1me9+fmcnIwVGEBhuJCYpiaGS6222vyVzA8KhMgo2u\nByH0Nk3V3O0fEkVFmmVapVcv4MvIV/rteNdkLuCNw286JaT2/Ya/N2wxT372tMOyZ3Ke4GjVMX6/\n688AhBhDgHKGRmXw49E/BOC/N9r6TL/xk0WUl1dzvLqYx3c+1WFMcwfPZEHaHJrGNLF8+2cA/HDs\nIv78+QtO284fkstVQ2Z1eE0gMSSB07WljIsbw61jbuj0+sENWdcwMCyFpWN+xN8P/ov3T35MqLGl\np4rjeQoxBPPEtIfaldBxs0yoKYRVl/7MYZlO0bF8wu2cb6xm5QeuL9jaXks8qyffA3Q+ZeW8IbOB\n1rb39j1KWhK0s85/nYyJHdnhdHOO7fit5a34wSXAJQ7b/r9Jyzo9niuzJ6Uye1Jq8zOFuyfc1ua9\ndTz3OalXkJPa9S+O/qapmntbqtTcvUZBcfnT3HmJ6w+8474tSa23MXV0xO6zt4F3sa2go+26NC16\nS73FR/tUd6YrNffusHp/6mefp8nkrugtGOJcz4Mq+p77D7DjcncfT4fUbm+q7e1FOlfb9DxBqNja\nY7t6kbTjrbrS5u6ZLzmv8XAy7sp1Ck/zt+9VbSX3Niff3STXonN6S9dubAk2BLtcnh45mAVD5jgt\nb//ZiAuKASAzMg2AcJNtJvfY5uUZkUO6FEdLH+W0iEFu9xkenQmAQacHbO23A5rblqMCHK9JZEa5\nb2cFW5NM9oBJAIyLH9PhtlMGXArAiLjWLnSjY2z9nS9PvgxwznuuEtcl8VkAXJp4idO6zgTpbc1Y\n7ftne5pB0btdd2WqrSviiOih9mVhze/3wLDkTsvWtys7Lth2TSQ9omv/I73R8n9q7OIdq75CUV1d\nzelnHbXxdUVLO+EbR96k6Ph7HorKs2ICYqho6PqMTtkDJrKnfD915nq32zyc/f+oajjPH3b9BYDr\nh13D2LhRWFUrDfUQFKhDr9Px1zePst9YiC7Y9Xl+esbj7D3zBQXNN4u8ds2f+OKbk4QE6SmuPsXA\n8BT+eWiTU1/s7w37DoPDB/Gv7cc4GFwIQP7IHzAx0ZaIiqtPERccy8ZDm/ioZCcmvYk/XPmoQxnV\njTUEG4KotzRg0pvsdwq2LP/Drmf55vy3DI/K5K7xtwLQ0GjBYlUZlBpl/99p2R7ggrmOUGMIxTWn\nSAlN4nxjNREB4fZj1jTVEqAPwKgzcL6xmlBjCNWNtYSZbH8jAmxJx6paOVVzmjU719r3fXDyz4kN\nikGn6BzKvdB0gfK6CpJDBzhcW7CqVqoba8lISXL4Pz/XcJ5wUxiKonDg7GH+tOc5+7pgQxC/nfaw\n0/t0ruG8w+vojprGWgINAZ2Ol9JZm3uLOnMdOkVPgN4EwIWmOgw6g9s7TVVV5Xxjjf3ctmh53/Q6\n918MtrL1mJqP1RLn1ydLOt3XE8xWM/WWhjbXSLqnq+e0p2W7o6kLqr4sITS2W8ldQUeYKbTD5B4b\nFGOv5TbvRJipuedGQLuNO/gKVxTF4WYVo8FAcqStZtRyq3t8m14rbaWGJRGoVtqfRwe13hqfEmbr\nDRPYPGiUq1+1LfGG6IJdLncVeIDJ+cPcun3r49TmGmH7hNj2Q9rya6El6bRNPjpFZ38NrWWH2ZNJ\n23KDjcEMMjr/ktEpOqeE5iqmttw1M/U0sYPtgqsnBRkce4h01mNEURSX56Ht++aOu7K7sq8nGHQG\nQrs4iJgv0UyzjKqqHd7x5m3WLt4Q0aIn7Xu9ucjU2YU6ly3WLrsu903DpK9cSOyLOLzRfiy0TzPJ\n/d3iD9l3xnfHKun+IERdTyItfcFbaqHtDYgJxnrB/c836Lw9McZFv+/I5ppkYnRrjTXE4Nk+v76Q\n9kLa1Mj75IYVX3iRQnP877eGG5+X7/fasQeFp/Lt+RMdbrMw/SreLf6wewV38UN/9/jb+KLiIGPd\n3EAyP3sw4fsXoo8uZePRfwGtgyu1XAQL0Ju4dcyPiA9yfWPP5AGXUN1Yw/6Kr5g9aDo1jbX2G1bm\nTh6E7vMbiYyvI8HFl5jaSR/tjvRmX09ZMeluHvjINtmErg9q7lbJ7qIPaCa5e+vD/+S0hwkyBHV6\nE0tAm4tBXaHQmtgmJozj09I9breNCAhnStIkt+tNRj2zxqcBafbkbm9Db5Osxrm4e7KFTtGRO3iG\ny4kGjAYd8y4Z5f7F2HNXD94jH8h7be+M7Jv/Mx94kUJzNNMs43edUDujKP3zke/HzlK9eot85O3t\nkzZ373dYExqkneSuYX1RW+zPC5QtvUu6OrPNxab98LjGbk5cIYQrGmqW6X/BarRTlzAAtcmEYmy0\nP7+yeSzpS1PGsaPYsXllYdpc/rt5HO6ogEgqG6qA5jsf29To5g/J5c1vtvQ65gVpc9GhUGdx38XS\n02YPnE7ZhTP2cUm6w1d6ktww4jqOVxd3eV7N7hgWlcGE+LGMjR3FrrK9PTpPQrTXq5r7Sy+9xLx5\n85g/fz733HMPDQ0NVFVVkZ+fT25uLvn5+Zw7d85TsXaoJ7XbppLBPT5e45EsxlsXOy1vODwO1Wo7\nrZcNmMgzOU9w3dBFAOQNd75rc3rq5fbHj05d1aZnRtvXo3DVkFlMS+79wP5zB+f02wS9LUJNIdw2\n9kanPuPd4e1hVS8bMJHrhub1Sdl6nZ5bRv+QSYnje32ehGjR4+ReWlrKyy+/zOuvv86bb76JxWJh\n8+bNFBQUkJ2dzZYtW8jOzqagoMCT8fo8RWmtaXYlITlto7p8eBHz8zFVhPCSXtXcLRYL9fX1mM1m\n6uvriY+Pp6ioiLw8Ww0nLy+PrVu1O0m1u3brlgTfk+Gs7F3/FNuz1setzy8mbb8qhRBd1+MGxISE\nBG6++WZmzJhBQEAAU6dO5fLLL6eiooL4eFtf57i4OCoqOr/lPioqGIOhd+ND1Nabu7W92tT7i1bB\nwSb72A5Xpkxje/F75E2cxNtfQdOAz8kdfoXD2A8hjbbTPW3QZCrrq9hXepDEhEgyogdz5Owx4uLC\nUHS2JBYUZOLaMfN5ducrzB0+jbi4MGYrU3nv5Mf28joaV6IzOfrJbPl2G9dn5bkspzdle9Li0XP4\n48cvMn/kDJ+Osyv8JVaJ0/O8EWuPk/u5c+coKiqiqKiIsLAw7r77bjZtcpxFXVFcj+vdXmXlhZ6G\nAdhOXGNT927vN5enNE/qYRv7vWHv5QRmvQ/AlSlTWJwxH6tq5Wfbf2HfZ1BYKt9Wt96sVFfXaB8Q\n6Lqh81k4JJdAo4mrxmVisX4Xo2JwGDAoLi6Mp6avQa/To6oq1hFWKs7UsjzrDqyqlfLyaqxWtbns\nJsakjrFvX15eTTTx3DPhJ/aJLHozGFEY0fxx+q8x6AxO5fTlQEfdNTRouMM5aMuX4uyMv8QqcXqe\ntwYO63GzzEcffURKSgrR0dEYjUZyc3PZvXs3MTExlJWVAVBWVkZ0dNenK/MaqwKq46mwjXDneONR\n+9Hn2n9vBRpt2+t0CkaD6+/NljIURXH5uH0bc/tj6nWe673aFz0/+kJfj/onhBb1OFMkJSXx+eef\nU1dXh6qqfPzxx6Snp5OTk0NhoW3418LCQmbOnOmxYDtSaS7twV6Kw58W7u4paf8bpG9uaGk5lpvJ\nlC++ZnchRA/0uOqWlZXFnDlz+M53voPBYGDEiBF897vfpba2luXLl7Nx40aSkpJYu3Zt54X1Ul1T\nPY10v9+22jyYVpg5mYY2y1NCB7jcPj1yCBfMdZTUlmJtCCI1ri+GHO14lvjwliFy3UyUIYQQ0Mub\nmJYtW8ayZY4T0ZpMJtavX9+roLqrwdzQ+UbtKWCpGMCP5mYxPDaNL9PKeLV4OwDZLsZp+UnWzQyL\nymDWwCs5WnkcZXAco4Z4vsmps4GyYoKiWT7+9h6MMimEuJj4R6NrZ3rcPKIwInoY4UEmhg+MguaZ\n+drfDg4wqnlqNIPOwNj44eDF3JoZlea9gwsh/MJFP7aMrzVh+1o8Qgj/pInk3qN6u8X2o6W5W7nP\n9MiIaJ5wI9jDk14IIS4ummiW6ew2/1gllTPqCWKDYjhTZ7up6o6p8zleUk9YsK37YrgpjKsHzyIt\ncrDDvtcPu6ZfE/9Psm5m24kPmZF6Rb8dUwihPZpI7p21uT884y4A/vrF3zhTV0FsUAzj0wcwPt1x\nu3lpuU77Tk2e7LEwuyI+OI7vDuubAaqEEBePi7JZRkYpEUJonUaSe9fSdcuMN94ePlYIIfqaJpJ7\nR7k6Mbi1z+LsQdPRKTqWNI+vLoQQWqWNNvcO+g8+cNnP7Y9Tw5L504zf9ENAQgjhXdqouQshhHCg\nieTuK/NsCiGEr9BEchdCCOFIE8ndXb19TMzofo1DCCF8hSaSuzsmvW8MKSCEEP1NG8ndzQwWVmmL\nF0JcpLSR3N2ICojwdghCCOEVmujn7qq3zFWDZzFr4DQvRCOEEN6nyZp7U8lg5qflEmgI9HYoQgjh\nFZpI7tKyLoQQjjSR3N1dUBVCiIuVNpJ7O5Yzyd4OQQghvEoTyb19vV01G70ShxBC+ApNJHchhBCO\nNJHc95Ue8HYIQgjhUzSR3J/+5CVvhyCEED5FE8ldCCGEI0nuQgihQZLchRBCg7SZ3K3afFlCCNFV\nmsyCj918hbdDEEIIr9Jkch8QE+LtEIQQwqs0mdyFEOJiJ8ldCCE0SJK7EEJoUK+S+/nz51m2bBlz\n587lqquuYvfu3VRVVZGfn09ubi75+fmcO3fOU7F2ibVGptYTQoheJffHHnuMK664grfeeotNmzaR\nnp5OQUEB2dnZbNmyhezsbAoKCjwVaxfJ2O5CCNHj5F5dXc3OnTtZsmQJACaTifDwcIqKisjLywMg\nLy+PrVu3eibSrlL693BCCOGLejxBdnFxMdHR0axcuZIDBw4watQoVq9eTUVFBfHx8QDExcVRUVHR\naVlRUcEYDPqehuIkLi7MY2V5mi/H1pbE6Xn+EqvE6XneiLXHyd1sNvPll1/ywAMPkJWVxaOPPurU\nBKMoCoqtNHoNAAATa0lEQVTSeVW6svJCT8NwQaW8vNqD5XlOXFyYz8bWlsTpef4Sq8TpeX0Za0df\nGj1ulklMTCQxMZGsrCwA5s6dy5dffklMTAxlZWUAlJWVER0d3dND9Iw0ywghRM+Te1xcHImJiXz9\n9dcAfPzxx6Snp5OTk0NhYSEAhYWFzJw50zORdplcUBVCiB43ywA88MAD/PznP6epqYnU1FTWrFmD\n1Wpl+fLlbNy4kaSkJNauXeupWN1SLXoUvQUAS1V8nx9PCCF8Xa+S+4gRI3jjjTeclq9fv743xXab\nWh+CEnKeppLBmIsz+/XYQgjhi3qV3H2JatFjPjHc22EIIYRPkOEHhBBCgyS5CyGEBmkkuUsPGSGE\naEsjyR1QpYO7EEK00EZyl7wuhBAOtJHcpVlGCCEcaCS5Q0v1PT0p3MtxCCGE92koudus/tFEb4cg\nhBBep53kLi0zQghh5/d3qFpVK7rgGm+HIYQQPsXva+67Sj/3dghCCOFz/D65n62vsj8emBDqxUiE\nEMJ3+H1yV9s0tocEGr0YiRBC+A6/T+5tdWFGPyGEuCj4fXJv20kmISrYa3EIIYQv8fvk3ja9L5me\n7sU4hBDCd2ggubcKCvD7np1CCOERfp/cVbl5SQghnPh9chdCCOFMA8ldqu5CCNGe3yd3VZK7EEI4\n8fvkLoQQwpnfJ3eptwshhDO/T+7SXUYIIZz5fXK3St1dCCGc+H9yt0pyF0KI9vw/uatWb4cghBA+\nx++Tu0Vq7kII4cTvk7s0ywghhDP/T+7SLCOEEE78PrlbrJLchRCiPb9P7lbp5y6EEE4kuQshhAb1\nOrlbLBby8vK47bbbAKiqqiI/P5/c3Fzy8/M5d+5cr4PsiFxQFUIIZ71O7i+//DLp6a3T2xUUFJCd\nnc2WLVvIzs6moKCgt4fokEUuqAohhJNeJffTp0/z7rvvsmTJEvuyoqIi8vLyAMjLy2Pr1q29i7AT\npy4U92n5Qgjhj3o16eivf/1r7rvvPmpra+3LKioqiI+PByAuLo6KiopOy4mKCsZg0PcohtKGEvvj\nuLiwHpXRn/whRpA4+4K/xCpxep43Yu1xct+2bRvR0dGMHj2aTz75xOU2iqKgKEqnZVVWXuhpGA7K\ny6s9Uk5fiYsL8/kYQeLsC/4Sq8TpeX0Za0dfGj1O7rt27eKdd97hvffeo6GhgZqaGn7+858TExND\nWVkZ8fHxlJWVER0d3dNDCCGE6KEet7nfe++9vPfee7zzzjv8/ve/57LLLuPJJ58kJyeHwsJCAAoL\nC5k5c6bHghVCCNE1Hu/nvnTpUj788ENyc3P56KOPWLp0qacPIYQQohO9uqDaYvLkyUyePBmAqKgo\n1q9f74lihRBC9JDf36EqhBDCmSR3IYTQIEnuQgihQZLchRBCgyS5CyGEBklyF0IIDZLkLoQQGiTJ\nXQghNEiSuxBCaJAkdyGE0CBJ7kIIoUGS3IUQQoMkuQshhAZJchdCCA2S5C6EEBokyV0IITRIkrsQ\nQmiQ3yd31er3L0EIITxOA5lR9XYAQgjhczSQ3IUQQrTn/8ld8XYAQgjhe/w/uUuzjBBCOPH75K5I\nzV0IIZz4fXIXQgjhTJK7EEJokCR3IYTQIEnuQgihQZLchRBCgzST3EMNYd4OQQghfIZmkvsNQ37s\n7RCEEMJnGLwdQG+pjQGoVh2hxhBvhyKEED5DIzV3BUXGIRBCCDv/T+6KCqqCXi/JXQghWvh/cseW\n3JNjpVlGCCFa9LjNvaSkhPvvv5+KigoUReG6667jxhtvpKqqip/97GecPHmS5ORk1q5dS0REhCdj\ndqDTgapTUGSQGSGEsOtxzV2v17NixQr+/e9/s2HDBv72t79x5MgRCgoKyM7OZsuWLWRnZ1NQUODJ\neJ2oIO3tQgjRTo+Te3x8PKNGjQIgNDSUtLQ0SktLKSoqIi8vD4C8vDy2bt3qmUjdaW5zF0II0coj\nXSGLi4v56quvyMrKoqKigvj4eADi4uKoqKjodP+oqGAMBn0Pj26ru8fF+cdNTBKnZ/lLnOA/sUqc\nnueNWHud3Gtra1m2bBmrVq0iNDTUYZ2idK0tvLLyQi8isE3WUV5e3Ysy+kdcXJjE6UH+Eif4T6wS\np+f1ZawdfWn0qrdMU1MTy5YtY8GCBeTm5gIQExNDWVkZAGVlZURHR/fmEJ1TVBRplhFC9LHq6mre\neOOfHinrhRfW8be/veKRstzpcc1dVVVWr15NWloa+fn59uU5OTkUFhaydOlSCgsLmTlzpkcCdRsH\nKmaLTLUnRH/5xztH2HmgzKNlThoez3U5GR4t09Nqaqr517/+yeLF13o7lC7pcXL/7LPP2LRpE0OH\nDmXRokUA3HPPPSxdupTly5ezceNGkpKSWLt2rceCbe/oyXMoClil5i6EppWUnOLee+9i1Kgx7Nu3\nl3HjxjJz5lW8+OI6KisrefDBXwHwxz/+jsbGBgICAlm16kEGDhzMhg2vcfToEVat+iVHjx7hoYdW\n8dxzL9PY2MCaNY9w6tRJAgICuf/+1WRkZPLCC+sICgrm+utvAOCGG67jiSfW8uyzf+LkyZPcdNP1\nTJo0mZ/+9G6nOHft+pQXXywgNDSUo0ePkpMzi3HjRvPii3+loaGBNWt+R3JyisM+d965lIyMoezZ\nswuLxczKlQ8ycuToXp+zHif3iRMncvDgQZfr1q9f3+OAuqOyuqH5kSR3IfrLdTkZXqllnzxZzK9+\n9TgrV6Zxxx35vP32W/z5zy/wwQfbeeWVv/KLXzzMM888h8FgYOfOT1i37hkee+y3XHvt97nrrtvY\nvn0bL7/8Ivfdt4rAwED+8penyMwcxpo1v+Ozz3by6KO/5KWX/ub2+Lfffhdff320w20Ajhw5xKuv\nbiQ8PJzrrltEZGQozz33Mv/4x3+xceMG7r77Xqd9Ghrqeemlv7Fnzy7WrHmEV175R6/Pl18PHKbT\nNTfHSM1dCM0bMCCJ9HTbl0pGRgZZWRNRFIW0tAxKSkqoqanh0Ucforj4OIqiYDabAdDpdKxa9Utu\nuun7LFy4mLFjxwGwd+8eHn30CQAuuWQS58+fo7a2ptdxDh8+ktjYWACSk1OYOnUqAOnpGeze/anL\nfWbNmgPAuHETqK2tpbq6mrCw3vWw8evhBxS/jl4I0R1Go9H+WKfT2Z/rdDosFjPPP/8sEyZM5JVX\n/sHjj/+BxsZG+/bFxScICgrmzJnyTo+j1+tRVav9edtyusJkMtkfK4pif64oChaLxeU+7XsVeuKO\ne79Oj/bXLzV3IS56NTU1xMXFAfDvf/+Pw/K1a3/L008XcP78ObZts91YmZU1nrfffguwtZVHREQQ\nEhLKgAFJHDp0AICDBw9QUnIKgODgYC5c6E23bfeKirYA8PnnewgNDXXqVt4T/p3c7dFLchfiYveD\nH/yIZ599hvz86x1qyE899TsWL76OgQMHsWLFAzz77NNUVp7l5puXcvDgV9x44/d49tmnWb36YQCm\nT8/h/Pnz/PCH1/H66xtITR0IQEREJGPGZHHDDdfxzDN/9GjsJlMA+fnX8+STv2bFigc8UqaiqqrX\n+xH2tIP/3m9KWPfNH7BUxvHsNfd5OCrP85cbLyROz/OXWCVOz+ss1jvvXMqddy5n+PCRPSrbHb+u\nuet0zTV2aZYRQggHft1bpl6tA0AXVuXlSIQQF5OjR4/wq1896LDMaDTy3HPd7wb+9NN9M3KuXyf3\nPZW2bkWKsXtXs4UQojfS0zM67e/ubX7dLGNRXXcrEkKIi51fJ3cfuBYshBA+yb+TO5LchRDCFf9O\n7lJzF0IIl/w7uUvNXQjRxp13LuXAgS/7rPwlSxZQVeUfvfP8ureMVNyF6H9vHHmT3WX7PFrm+Pgx\nLM6Y36VtVVXFarV2vuFFzr+TO/IGC3ExKCk5xT333MnIkaM5ePAAt9++lFdeeY2mpkaSklJYteqX\nBAcHO+wze/YVvP32+wBs27aVjz76gNWrH3JZ/mOPPURAQACHDh2ksrKSlSsf4K23NvPFF/sYOXK0\n034t48sPGzaCQ4cOMGRIGr/4xSMEBgb2xcvvEb9O7jqL75xIIS4WizPmd7mW7UnFxSdYvfphUlJS\neeihFaxd+2eCgoJ49dWX2LDhNfLzb+1V+dXV51m37q988MF2Vqy4l7/85QWGDEnjxz/+EYcPHyQz\nc5jD9sePf8uKFQ8wduw4fv3rh3njjX/aJ/jwBX7d5n7Z4OEAzBp8pZcjEUL0tcTEAYwePYYvvtjH\nkSNHuOOOW7jpput5663NnD5d0uvyp06dZh8fPjo6mvT0DHQ6HUOGpFFS4lx+fHyCfWz4OXOuZt++\nPb2OwZP8uuaubx5bJiUq3suRCCH6WkuTh6qqTJ06lZUrH+5kj9Yxp7oyJnvb8eHbjx1vsZidS3ca\nc923xrjy65p7y/VUxcdOqhCi74waNYZdu3ZRXHwCgLq6Oo4f/9Zpu+joaI4d+war1cp7723zeByl\npafZv38vAG+//Za9Fu8r/Lrm3tIV0hOzlggh/ENUVBRr1qzhoYdW09Rkq5HfeusdDBw4yGG722+/\nk/vvX05kZBTDh4+grq7Oo3EMHDiIN974J2vWPMLgwUP4zneWeLT83vLr8dyrGs6x4WAhN09agrEh\nxMNReZ6/jEEtcXqev8QqcXZNSckp7r9/eZcmsu7LWDU7nntkQAS3jb2RpPBEb4cihBA+xa+bZYQQ\nojvWr3+BbduKHJbNmDGTG2+8pVvlDBiQ1KVauzdJchdCXDRuvPGWbidyf+XXzTJCCCFck+QuhBAa\nJMldCCE0SJK7EEJokCR3IYTQIEnuQgihQT5xh6oQQgjPkpq7EEJokCR3IYTQIEnuQgihQZLchRBC\ngyS5CyGEBklyF0IIDZLkLoQQGuTXyf29995jzpw5zJ49m4KCAm+HQ05ODgsWLGDRokUsXrwYgKqq\nKvLz88nNzSU/P59z587Zt1+3bh2zZ89mzpw5vP/++30W18qVK8nOzmb+/Pn2ZT2Ja//+/SxYsIDZ\ns2fz6KOP0he3SLiK9U9/+hNXXHEFixYtYtGiRWzfvt3rsZaUlHDDDTdw9dVXM2/ePNavXw/43nl1\nF6evndOGhgaWLFnCwoULmTdvHk899RTge+ezo1h97Zyi+imz2azOnDlTPX78uNrQ0KAuWLBAPXz4\nsFdjmjFjhlpRUeGw7PHHH1fXrVunqqqqrlu3Tn3iiSdUVVXVw4cPqwsWLFAbGhrU48ePqzNnzlTN\nZnOfxLVjxw51//796rx583oV1zXXXKPu3r1btVqt6i233KK+++67/RLrU089pT7//PNO23oz1tLS\nUnX//v2qqqpqdXW1mpubqx4+fNjnzqu7OH3tnFqtVrWmpkZVVVVtbGxUlyxZou7evdvnzmdHsfra\nOfXbmvvevXsZNGgQqampmEwm5s2bR1FRUec79rOioiLy8vIAyMvLY+vWrfbl8+bNw2QykZqayqBB\ng9i7d2+fxDBp0iQiIiJ6FVdZWRk1NTWMGzcORVHIy8vrk/PtKlZ3vBlrfHw8o0aNAiA0NJS0tDRK\nS0t97ry6i9Mdb8WpKAohIbZ5kM1mM2azGUVRfO58dhSrO96K1W+Te2lpKYmJrXOnJiQkdPhP21/y\n8/NZvHgxGzZsAKCiooL4+HgA4uLiqKioALwff3fjar88MTGxX+N99dVXWbBgAStXrrT/NPeVWIuL\ni/nqq6/Iysry6fPaNk7wvXNqsVhYtGgRU6ZMYcqUKT59Pl3FCr51Tv02ufui//qv/2LTpk0899xz\nvPbaa+zcudNhvaIoHX7De4uvxtXi+9//Plu3bmXTpk3Ex8fzm9/8xtsh2dXW1rJs2TJWrVpFaGio\nwzpfOq/t4/TFc6rX69m0aRPbt29n7969HDp0yGG9L51PV7H62jn12+SekJDA6dOn7c9LS0tJSEjw\nYkTYjx8TE8Ps2bPZu3cvMTExlJWVAVBWVkZ0dLR9W2/G39242i8/ffp0v8UbGxuLXq9Hp9Nx7bXX\nsm/fPp+ItampiWXLlrFgwQJyc3MB3zyvruL01XMKEB4ezuTJk3n//fd98ny6i9XXzqnfJvcxY8Zw\n7NgxTpw4QWNjI5s3byYnJ8dr8Vy4cIGamhr74w8//JDMzExycnIoLCwEoLCwkJkzZwK2njWbN2+m\nsbGREydOcOzYMcaOHdtv8XY3rvj4eEJDQ9mzZw+qqjrs09daPtwAW7duJTMz0+uxqqrK6tWrSUtL\nIz8/377c186ruzh97ZyePXuW8+fPA1BfX89HH31EWlqaz53PjmL1tXNq8FhJ/cxgMPDggw/y4x//\nGIvFwjXXXGM/md5QUVHBT3/6U8DWHjd//nymTZvGmDFjWL58ORs3biQpKYm1a9cCkJmZyVVXXcXV\nV1+NXq/nwQcfRK/X90ls99xzDzt27KCyspJp06Zx1113sXTp0m7H9ctf/pKVK1dSX1/PtGnTmDZt\nWr/EumPHDg4cOABAcnIyjzzyiNdj/eyzz9i0aRNDhw5l0aJF9th97by6i/PNN9/0qXNaVlbGihUr\nsFgsqKrK3LlzmTFjBuPGjfOp89lRrPfdd59PnVMZz10IITTIb5tlhBBCuCfJXQghNEiSuxBCaJAk\ndyGE0CBJ7kIIoUGS3IUQQoMkuQshhAb9fyv4kl/mzbCKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fbe254022e8>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "plot1, = plt.plot(maxout_mlp_acc, label='maxout_mlp')\n",
    "plot2, = plt.plot(relu_mlp_acc, label='relu_mlp')\n",
    "plt.legend(handles=[plot1, plot2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}