SGD_DL_MiniProject.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "id": "G9zo1ZNgMQzb"
   },
   "outputs": [],
   "source": [
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "import torch.nn.functional as F\n",
    "import torch.backends.cudnn as cudnn\n",
    "import torchvision\n",
    "import torchvision.transforms as transforms\n",
    "import os\n",
    "import argparse\n",
    "from finalmodel import ResNet18, ResNet, BasicBlock\n",
    "from torchsummary import summary\n",
    "import random\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "id": "hufiv6CxiA7a"
   },
   "outputs": [],
   "source": [
    "SEED = 7772\n",
    "torch.manual_seed(SEED)\n",
    "random.seed(SEED)\n",
    "np.random.seed(SEED)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "id": "mBQus23bMQzd"
   },
   "outputs": [],
   "source": [
    "from functools import reduce"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "id": "qGme79fHMQzd"
   },
   "outputs": [],
   "source": [
    "device = 'cuda' if torch.cuda.is_available() else 'cpu'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "id": "P93Rse94jVQ0"
   },
   "outputs": [],
   "source": [
    "def build_model(sizes):\n",
    "  return ResNet(BasicBlock, sizes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7vQBgkF6idhX",
    "outputId": "6946df65-586d-45a6-dc5b-5124bc5db7c9"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0, 0, 0, 0]\n",
      "params:  4903242\n",
      "\n",
      "[0, 0, 0, 1]\n",
      "params:  4903242\n",
      "\n",
      "[0, 0, 1, 0]\n",
      "params:  4903242\n",
      "\n",
      "[0, 0, 1, 1]\n",
      "params:  4903242\n",
      "\n",
      "[0, 1, 0, 0]\n",
      "params:  4903242\n",
      "\n",
      "[0, 1, 0, 1]\n",
      "params:  4903242\n",
      "\n",
      "[0, 1, 1, 0]\n",
      "params:  4903242\n",
      "\n",
      "[0, 1, 1, 1]\n",
      "params:  4903242\n",
      "\n",
      "[1, 0, 0, 0]\n",
      "params:  4903242\n",
      "\n",
      "[1, 0, 0, 1]\n",
      "params:  4903242\n",
      "\n",
      "[1, 0, 1, 0]\n",
      "params:  4903242\n",
      "\n",
      "[1, 0, 1, 1]\n",
      "params:  4903242\n",
      "\n",
      "[1, 1, 0, 0]\n",
      "params:  4903242\n",
      "\n",
      "[1, 1, 0, 1]\n",
      "params:  4903242\n",
      "\n",
      "[1, 1, 1, 0]\n",
      "params:  4903242\n",
      "\n",
      "[1, 1, 1, 1]\n",
      "params:  4903242\n",
      "\n",
      "[2, 0, 0, 0]\n",
      "params:  4977226\n",
      "\n",
      "[2, 0, 0, 1]\n",
      "params:  4977226\n",
      "\n",
      "[2, 0, 1, 0]\n",
      "params:  4977226\n",
      "\n",
      "[2, 0, 1, 1]\n",
      "params:  4977226\n",
      "\n",
      "[2, 1, 0, 0]\n",
      "params:  4977226\n",
      "\n",
      "[2, 1, 0, 1]\n",
      "params:  4977226\n",
      "\n",
      "[2, 1, 1, 0]\n",
      "params:  4977226\n",
      "\n",
      "[2, 1, 1, 1]\n",
      "params:  4977226\n",
      "\n"
     ]
    }
   ],
   "source": [
    "N = 1\n",
    "B = [2] * N\n",
    "C = [32, 64, 128, 256, 512] \n",
    "F_i = 3\n",
    "\n",
    "P = 1\n",
    "PARAM_COUNT = 5_000_000 #5 million\n",
    "\n",
    "from collections import defaultdict\n",
    "good_models = defaultdict(list)\n",
    "from itertools import permutations, product\n",
    "for n in range(1, N+1):\n",
    "    perms = product([0,1,2,4,8], repeat=4)\n",
    "    for p in perms:\n",
    "      B = list(p)\n",
    "      model = build_model(B).to(device)\n",
    "      total_params = sum(p.numel() for p in model.parameters() if p.requires_grad)\n",
    "      if total_params <= PARAM_COUNT:\n",
    "          print(B)\n",
    "          print(\"params: \", total_params)\n",
    "          print()\n",
    "          good_models[total_params].append(model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "7mkWOEI-j5-C",
    "outputId": "4cda66db-e455-4e9f-cb55-60bcc2479fc6"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Total Good Models:  2\n"
     ]
    }
   ],
   "source": [
    "print(\"Total Good Models: \", len(good_models.keys()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 102,
     "referenced_widgets": [
      "f367da63ac8e4cdfa061246e17453d3d",
      "3e4fbcce4b2c4d3284eca06b7ea2c9a6",
      "8f660bd297b44104bf3db721bc818370",
      "8d10a341d7ff44078bfece90053cfe98",
      "a327fc83f72c47a0a038163048052c43",
      "fcb0c93edfd64a0fafa6cc57a0e279f1",
      "e8499808989a4bb8a71e8d8b45ad0683",
      "d16433ad0a68403c911ac4b2e42a73e4",
      "418f9f3ec8a0421a89c33d8f6fea4b7e",
      "9abe1b6a23a341fba09ca1f2e8e106d2",
      "ac7d242919984cef96d3a72a51916f92"
     ]
    },
    "id": "5XMd57ZSMQze",
    "outputId": "5b9e59ec-53d2-4891-a5fc-6dc6db5aa420"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Files already downloaded and verified\n",
      "Files already downloaded and verified\n"
     ]
    }
   ],
   "source": [
    "transform_train = transforms.Compose([\n",
    "    transforms.RandomCrop(32, padding=4),\n",
    "    transforms.RandomHorizontalFlip(),\n",
    "    transforms.RandomRotation(20),\n",
    "    transforms.RandomCrop(size=32, padding=[0, 2, 3, 4]),\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "])\n",
    "\n",
    "transform_test = transforms.Compose([\n",
    "    transforms.ToTensor(),\n",
    "    transforms.Normalize((0.4914, 0.4822, 0.4465), (0.2023, 0.1994, 0.2010)),\n",
    "])\n",
    "\n",
    "trainset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=True, download=True, transform=transform_train)\n",
    "trainloader = torch.utils.data.DataLoader(\n",
    "    trainset, batch_size=64, shuffle=True, num_workers=2)\n",
    "\n",
    "testset = torchvision.datasets.CIFAR10(\n",
    "    root='./data', train=False, download=True, transform=transform_test)\n",
    "testloader = torch.utils.data.DataLoader(\n",
    "    testset, batch_size=50, shuffle=False, num_workers=2)\n",
    "\n",
    "classes = ('plane', 'car', 'bird', 'cat', 'deer',\n",
    "           'dog', 'frog', 'horse', 'ship', 'truck')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "hMabamXfMQzf",
    "outputId": "d1f54c1c-4472-42e4-8123-1642383e26e7"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 32, 32]) 6\n"
     ]
    }
   ],
   "source": [
    "image, label = trainset[0]\n",
    "print(image.shape, label)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "id": "aPPl3UcJMQzg"
   },
   "outputs": [],
   "source": [
    "net = ResNet18()\n",
    "net = net.to(device)\n",
    "if device == 'cuda':\n",
    "    net = torch.nn.DataParallel(net)\n",
    "    cudnn.benchmark = True\n",
    "    \n",
    "criterion = nn.CrossEntropyLoss()\n",
    "optimizer = optim.SGD(net.parameters(), lr=.01,\n",
    "                      momentum=0.9, weight_decay=5e-4)\n",
    "scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "um3K2Q_iMQzg",
    "outputId": "3c184ece-93e2-4467-fd7f-3bd7f3a3b9f8"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "----------------------------------------------------------------\n",
      "        Layer (type)               Output Shape         Param #\n",
      "================================================================\n",
      "            Conv2d-1           [-1, 64, 32, 32]           1,728\n",
      "       BatchNorm2d-2           [-1, 64, 32, 32]             128\n",
      "            Conv2d-3           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-4           [-1, 64, 32, 32]             128\n",
      "            Conv2d-5           [-1, 64, 32, 32]          36,864\n",
      "       BatchNorm2d-6           [-1, 64, 32, 32]             128\n",
      "        BasicBlock-7           [-1, 64, 32, 32]               0\n",
      "            Conv2d-8          [-1, 128, 16, 16]          73,728\n",
      "       BatchNorm2d-9          [-1, 128, 16, 16]             256\n",
      "           Conv2d-10          [-1, 128, 16, 16]         147,456\n",
      "      BatchNorm2d-11          [-1, 128, 16, 16]             256\n",
      "           Conv2d-12          [-1, 128, 16, 16]           8,192\n",
      "      BatchNorm2d-13          [-1, 128, 16, 16]             256\n",
      "       BasicBlock-14          [-1, 128, 16, 16]               0\n",
      "           Conv2d-15            [-1, 256, 8, 8]         294,912\n",
      "      BatchNorm2d-16            [-1, 256, 8, 8]             512\n",
      "           Conv2d-17            [-1, 256, 8, 8]         589,824\n",
      "      BatchNorm2d-18            [-1, 256, 8, 8]             512\n",
      "           Conv2d-19            [-1, 256, 8, 8]          32,768\n",
      "      BatchNorm2d-20            [-1, 256, 8, 8]             512\n",
      "       BasicBlock-21            [-1, 256, 8, 8]               0\n",
      "           Conv2d-22            [-1, 512, 4, 4]       1,179,648\n",
      "      BatchNorm2d-23            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-24            [-1, 512, 4, 4]       2,359,296\n",
      "      BatchNorm2d-25            [-1, 512, 4, 4]           1,024\n",
      "           Conv2d-26            [-1, 512, 4, 4]         131,072\n",
      "      BatchNorm2d-27            [-1, 512, 4, 4]           1,024\n",
      "       BasicBlock-28            [-1, 512, 4, 4]               0\n",
      "           Linear-29                   [-1, 10]           5,130\n",
      "           ResNet-30                   [-1, 10]               0\n",
      "================================================================\n",
      "Total params: 4,903,242\n",
      "Trainable params: 4,903,242\n",
      "Non-trainable params: 0\n",
      "----------------------------------------------------------------\n",
      "Input size (MB): 0.01\n",
      "Forward/backward pass size (MB): 6.56\n",
      "Params size (MB): 18.70\n",
      "Estimated Total Size (MB): 25.28\n",
      "----------------------------------------------------------------\n"
     ]
    }
   ],
   "source": [
    "summary(net,image.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "MXGyS6yIMQzi",
    "outputId": "cf6263c6-74f0-4775-eec6-184c7e8e0056"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Epoch: 0\n",
      "Epoch 0, Train loss 1185.9647044539452, Test loss 261.0464889407158, Accuracy 53.98\n",
      "\n",
      "Epoch: 1\n",
      "Epoch 1, Train loss 895.8092020750046, Test loss 194.70115292072296, Accuracy 65.36\n",
      "\n",
      "Epoch: 2\n",
      "Epoch 2, Train loss 766.838760137558, Test loss 169.3367090523243, Accuracy 70.52\n",
      "\n",
      "Epoch: 3\n",
      "Epoch 3, Train loss 673.1606594026089, Test loss 163.67675477266312, Accuracy 71.61\n",
      "\n",
      "Epoch: 4\n",
      "Epoch 4, Train loss 609.4876143634319, Test loss 158.03608870506287, Accuracy 73.09\n",
      "\n",
      "Epoch: 5\n",
      "Epoch 5, Train loss 563.0885302722454, Test loss 121.65910959243774, Accuracy 79.39\n",
      "\n",
      "Epoch: 6\n",
      "Epoch 6, Train loss 530.399913996458, Test loss 115.54643373191357, Accuracy 80.15\n",
      "\n",
      "Epoch: 7\n",
      "Epoch 7, Train loss 491.0487681925297, Test loss 110.61277632415295, Accuracy 80.36\n",
      "\n",
      "Epoch: 8\n",
      "Epoch 8, Train loss 464.2114199101925, Test loss 125.1091413795948, Accuracy 78.97\n",
      "\n",
      "Epoch: 9\n",
      "Epoch 9, Train loss 444.3897643685341, Test loss 122.83037221431732, Accuracy 78.85\n",
      "\n",
      "Epoch: 10\n",
      "Epoch 10, Train loss 425.78731332719326, Test loss 112.6114621013403, Accuracy 80.84\n",
      "\n",
      "Epoch: 11\n",
      "Epoch 11, Train loss 409.71262806653976, Test loss 92.88462966680527, Accuracy 84.34\n",
      "\n",
      "Epoch: 12\n",
      "Epoch 12, Train loss 388.8839582800865, Test loss 91.79595457017422, Accuracy 84.46\n",
      "\n",
      "Epoch: 13\n",
      "Epoch 13, Train loss 377.78543481230736, Test loss 98.07231459021568, Accuracy 83.58\n",
      "\n",
      "Epoch: 14\n",
      "Epoch 14, Train loss 365.37913878262043, Test loss 89.49496871978045, Accuracy 85.17\n",
      "\n",
      "Epoch: 15\n",
      "Epoch 15, Train loss 349.9939545840025, Test loss 90.15120550990105, Accuracy 85.12\n",
      "\n",
      "Epoch: 16\n",
      "Epoch 16, Train loss 339.98043133318424, Test loss 101.55921267718077, Accuracy 83.0\n",
      "\n",
      "Epoch: 17\n",
      "Epoch 17, Train loss 334.34311175346375, Test loss 77.68655291944742, Accuracy 87.04\n",
      "\n",
      "Epoch: 18\n",
      "Epoch 18, Train loss 325.90279534459114, Test loss 90.12878705561161, Accuracy 84.93\n",
      "\n",
      "Epoch: 19\n",
      "Epoch 19, Train loss 313.51320973038673, Test loss 83.13031448423862, Accuracy 85.95\n",
      "\n",
      "Epoch: 20\n",
      "Epoch 20, Train loss 305.5278248935938, Test loss 85.24476306140423, Accuracy 85.86\n",
      "\n",
      "Epoch: 21\n",
      "Epoch 21, Train loss 299.71600310504436, Test loss 79.54623925685883, Accuracy 86.83\n",
      "\n",
      "Epoch: 22\n",
      "Epoch 22, Train loss 296.79074916243553, Test loss 83.47080419957638, Accuracy 86.36\n",
      "\n",
      "Epoch: 23\n",
      "Epoch 23, Train loss 285.0579056739807, Test loss 75.82342460006475, Accuracy 87.16\n",
      "\n",
      "Epoch: 24\n",
      "Epoch 24, Train loss 281.70570716261864, Test loss 78.04475901275873, Accuracy 87.15\n",
      "\n",
      "Epoch: 25\n",
      "Epoch 25, Train loss 274.47343088686466, Test loss 74.21558416634798, Accuracy 87.83\n",
      "\n",
      "Epoch: 26\n",
      "Epoch 26, Train loss 272.0322286337614, Test loss 77.98556668311357, Accuracy 87.26\n",
      "\n",
      "Epoch: 27\n",
      "Epoch 27, Train loss 267.6687867715955, Test loss 75.53065042197704, Accuracy 87.32\n",
      "\n",
      "Epoch: 28\n",
      "Epoch 28, Train loss 263.20160380750895, Test loss 77.00066514313221, Accuracy 87.33\n",
      "\n",
      "Epoch: 29\n",
      "Epoch 29, Train loss 259.8713703677058, Test loss 83.52251753211021, Accuracy 86.5\n",
      "\n",
      "Epoch: 30\n",
      "Epoch 30, Train loss 254.55470944941044, Test loss 86.04208846390247, Accuracy 85.68\n",
      "\n",
      "Epoch: 31\n",
      "Epoch 31, Train loss 252.06249617040157, Test loss 82.7900557294488, Accuracy 87.01\n",
      "\n",
      "Epoch: 32\n",
      "Epoch 32, Train loss 249.25263233482838, Test loss 67.51461543887854, Accuracy 88.87\n",
      "\n",
      "Epoch: 33\n",
      "Epoch 33, Train loss 244.73514558374882, Test loss 69.43814515322447, Accuracy 88.69\n",
      "\n",
      "Epoch: 34\n",
      "Epoch 34, Train loss 244.48829247802496, Test loss 87.39872080087662, Accuracy 86.18\n",
      "\n",
      "Epoch: 35\n",
      "Epoch 35, Train loss 242.54535607993603, Test loss 66.84493127465248, Accuracy 89.12\n",
      "\n",
      "Epoch: 36\n",
      "Epoch 36, Train loss 239.54269990324974, Test loss 69.91835736483335, Accuracy 88.15\n",
      "\n",
      "Epoch: 37\n",
      "Epoch 37, Train loss 233.8577607497573, Test loss 70.38007157295942, Accuracy 88.34\n",
      "\n",
      "Epoch: 38\n",
      "Epoch 38, Train loss 237.1094383150339, Test loss 75.87121757119894, Accuracy 87.91\n",
      "\n",
      "Epoch: 39\n",
      "Epoch 39, Train loss 231.59339480102062, Test loss 65.56400660425425, Accuracy 89.11\n",
      "\n",
      "Epoch: 40\n",
      "Epoch 40, Train loss 228.43680381029844, Test loss 70.31766215711832, Accuracy 88.46\n",
      "\n",
      "Epoch: 41\n",
      "Epoch 41, Train loss 228.82022564858198, Test loss 65.08359413594007, Accuracy 89.36\n",
      "\n",
      "Epoch: 42\n",
      "Epoch 42, Train loss 226.82261347025633, Test loss 70.64833123981953, Accuracy 88.64\n",
      "\n",
      "Epoch: 43\n",
      "Epoch 43, Train loss 224.877517208457, Test loss 71.69319012761116, Accuracy 88.44\n",
      "\n",
      "Epoch: 44\n",
      "Epoch 44, Train loss 222.4286996871233, Test loss 74.66258361190557, Accuracy 87.81\n",
      "\n",
      "Epoch: 45\n",
      "Epoch 45, Train loss 223.2396097034216, Test loss 72.15073906630278, Accuracy 88.6\n",
      "\n",
      "Epoch: 46\n",
      "Epoch 46, Train loss 224.28307297825813, Test loss 70.74581500142813, Accuracy 88.1\n",
      "\n",
      "Epoch: 47\n",
      "Epoch 47, Train loss 217.93422374874353, Test loss 76.50139239430428, Accuracy 87.9\n",
      "\n",
      "Epoch: 48\n",
      "Epoch 48, Train loss 219.50507654249668, Test loss 71.01472783088684, Accuracy 88.1\n",
      "\n",
      "Epoch: 49\n",
      "Epoch 49, Train loss 213.19654205068946, Test loss 76.23955070227385, Accuracy 87.75\n",
      "\n",
      "Epoch: 50\n",
      "Epoch 50, Train loss 213.32534264028072, Test loss 84.39509989321232, Accuracy 86.69\n",
      "\n",
      "Epoch: 51\n",
      "Epoch 51, Train loss 213.11791701614857, Test loss 75.52330920100212, Accuracy 88.15\n",
      "\n",
      "Epoch: 52\n",
      "Epoch 52, Train loss 207.4186784401536, Test loss 71.12685140967369, Accuracy 89.12\n",
      "\n",
      "Epoch: 53\n",
      "Epoch 53, Train loss 206.75119984149933, Test loss 78.43194018304348, Accuracy 87.61\n",
      "\n",
      "Epoch: 54\n",
      "Epoch 54, Train loss 209.4470065459609, Test loss 71.79485120624304, Accuracy 88.48\n",
      "\n",
      "Epoch: 55\n",
      "Epoch 55, Train loss 205.81885761767626, Test loss 69.82764369249344, Accuracy 88.92\n",
      "\n",
      "Epoch: 56\n",
      "Epoch 56, Train loss 206.88343822211027, Test loss 79.7955304980278, Accuracy 87.87\n",
      "\n",
      "Epoch: 57\n",
      "Epoch 57, Train loss 211.71464018523693, Test loss 75.57516057789326, Accuracy 87.82\n",
      "\n",
      "Epoch: 58\n",
      "Epoch 58, Train loss 204.97101102024317, Test loss 76.03248065710068, Accuracy 87.67\n",
      "\n",
      "Epoch: 59\n",
      "Epoch 59, Train loss 203.2848520874977, Test loss 73.64579600095749, Accuracy 87.99\n",
      "\n",
      "Epoch: 60\n",
      "Epoch 60, Train loss 201.44982463866472, Test loss 74.08427197486162, Accuracy 88.37\n",
      "\n",
      "Epoch: 61\n",
      "Epoch 61, Train loss 199.50599117577076, Test loss 83.88416766375303, Accuracy 86.76\n",
      "\n",
      "Epoch: 62\n",
      "Epoch 62, Train loss 201.8912151120603, Test loss 76.49718601256609, Accuracy 88.13\n",
      "\n",
      "Epoch: 63\n",
      "Epoch 63, Train loss 200.03152964264154, Test loss 62.89205129444599, Accuracy 89.55\n",
      "\n",
      "Epoch: 64\n",
      "Epoch 64, Train loss 199.37836830317974, Test loss 65.81714006513357, Accuracy 89.4\n",
      "\n",
      "Epoch: 65\n",
      "Epoch 65, Train loss 195.15550546348095, Test loss 68.30761963129044, Accuracy 89.14\n",
      "\n",
      "Epoch: 66\n",
      "Epoch 66, Train loss 199.39892753213644, Test loss 74.26391891390085, Accuracy 88.15\n",
      "\n",
      "Epoch: 67\n",
      "Epoch 67, Train loss 199.8868248835206, Test loss 68.8885694667697, Accuracy 89.5\n",
      "\n",
      "Epoch: 68\n",
      "Epoch 68, Train loss 192.32089206576347, Test loss 67.4955081269145, Accuracy 89.02\n",
      "\n",
      "Epoch: 69\n",
      "Epoch 69, Train loss 191.4247326143086, Test loss 64.00380303710699, Accuracy 89.47\n",
      "\n",
      "Epoch: 70\n",
      "Epoch 70, Train loss 195.2052352577448, Test loss 68.10843110084534, Accuracy 89.69\n",
      "\n",
      "Epoch: 71\n",
      "Epoch 71, Train loss 191.98623728007078, Test loss 74.60704327374697, Accuracy 88.04\n",
      "\n",
      "Epoch: 72\n",
      "Epoch 72, Train loss 193.51982368156314, Test loss 73.69055836647749, Accuracy 88.53\n",
      "\n",
      "Epoch: 73\n",
      "Epoch 73, Train loss 191.9812226369977, Test loss 71.06481105089188, Accuracy 88.76\n",
      "\n",
      "Epoch: 74\n",
      "Epoch 74, Train loss 189.63369829952717, Test loss 67.48800726234913, Accuracy 89.3\n",
      "\n",
      "Epoch: 75\n",
      "Epoch 75, Train loss 190.2138964124024, Test loss 81.37871526554227, Accuracy 87.53\n",
      "\n",
      "Epoch: 76\n",
      "Epoch 76, Train loss 189.62834645062685, Test loss 73.458176381886, Accuracy 88.78\n",
      "\n",
      "Epoch: 77\n",
      "Epoch 77, Train loss 186.3237293213606, Test loss 72.3324436917901, Accuracy 88.42\n",
      "\n",
      "Epoch: 78\n",
      "Epoch 78, Train loss 191.8911216557026, Test loss 75.28668437153101, Accuracy 87.99\n",
      "\n",
      "Epoch: 79\n",
      "Epoch 79, Train loss 184.14567520841956, Test loss 64.14110294729471, Accuracy 89.69\n",
      "\n",
      "Epoch: 80\n",
      "Epoch 80, Train loss 188.65899378806353, Test loss 77.4222097992897, Accuracy 87.97\n",
      "\n",
      "Epoch: 81\n",
      "Epoch 81, Train loss 189.6177994608879, Test loss 80.99928008764982, Accuracy 87.38\n",
      "\n",
      "Epoch: 82\n",
      "Epoch 82, Train loss 184.67521370202303, Test loss 66.95717986673117, Accuracy 89.22\n",
      "\n",
      "Epoch: 83\n",
      "Epoch 83, Train loss 185.23638982325792, Test loss 73.64636093378067, Accuracy 88.95\n",
      "\n",
      "Epoch: 84\n",
      "Epoch 84, Train loss 188.71964357420802, Test loss 72.36906754225492, Accuracy 88.4\n",
      "\n",
      "Epoch: 85\n",
      "Epoch 85, Train loss 187.0237705335021, Test loss 79.67663144692779, Accuracy 87.09\n",
      "\n",
      "Epoch: 86\n",
      "Epoch 86, Train loss 186.44588074088097, Test loss 74.4811496809125, Accuracy 88.49\n",
      "\n",
      "Epoch: 87\n",
      "Epoch 87, Train loss 185.96516144275665, Test loss 65.38986616581678, Accuracy 89.75\n",
      "\n",
      "Epoch: 88\n",
      "Epoch 88, Train loss 181.60126937180758, Test loss 68.78156142681837, Accuracy 88.88\n",
      "\n",
      "Epoch: 89\n",
      "Epoch 89, Train loss 182.19109793379903, Test loss 69.62748145312071, Accuracy 89.16\n",
      "\n",
      "Epoch: 90\n",
      "Epoch 90, Train loss 178.19765546545386, Test loss 70.5928099155426, Accuracy 88.56\n",
      "\n",
      "Epoch: 91\n",
      "Epoch 91, Train loss 182.55492026358843, Test loss 70.3721631988883, Accuracy 88.78\n",
      "\n",
      "Epoch: 92\n",
      "Epoch 92, Train loss 177.11339815706015, Test loss 70.321554210037, Accuracy 89.16\n",
      "\n",
      "Epoch: 93\n",
      "Epoch 93, Train loss 183.06605499982834, Test loss 71.65071119368076, Accuracy 88.75\n",
      "\n",
      "Epoch: 94\n",
      "Epoch 94, Train loss 180.79181188717484, Test loss 71.74321273341775, Accuracy 88.74\n",
      "\n",
      "Epoch: 95\n",
      "Epoch 95, Train loss 178.6236548423767, Test loss 65.34817086905241, Accuracy 89.83\n",
      "\n",
      "Epoch: 96\n",
      "Epoch 96, Train loss 178.0735860839486, Test loss 70.7020586580038, Accuracy 89.12\n",
      "\n",
      "Epoch: 97\n",
      "Epoch 97, Train loss 180.4800181426108, Test loss 71.79850011691451, Accuracy 88.82\n",
      "\n",
      "Epoch: 98\n",
      "Epoch 98, Train loss 177.18191437795758, Test loss 66.68171620741487, Accuracy 89.72\n",
      "\n",
      "Epoch: 99\n",
      "Epoch 99, Train loss 180.74107342213392, Test loss 62.99939203262329, Accuracy 89.92\n",
      "\n",
      "Epoch: 100\n",
      "Epoch 100, Train loss 180.71635925024748, Test loss 61.59382054582238, Accuracy 89.96\n",
      "\n",
      "Epoch: 101\n",
      "Epoch 101, Train loss 176.9779295772314, Test loss 68.22582575678825, Accuracy 89.62\n",
      "\n",
      "Epoch: 102\n",
      "Epoch 102, Train loss 173.62554473802447, Test loss 75.83724450320005, Accuracy 88.48\n",
      "\n",
      "Epoch: 103\n",
      "Epoch 103, Train loss 174.22425619885325, Test loss 76.27475776523352, Accuracy 88.39\n",
      "\n",
      "Epoch: 104\n",
      "Epoch 104, Train loss 177.10878647863865, Test loss 68.97421892359853, Accuracy 88.9\n",
      "\n",
      "Epoch: 105\n",
      "Epoch 105, Train loss 179.2859203852713, Test loss 63.92699274420738, Accuracy 89.99\n",
      "\n",
      "Epoch: 106\n",
      "Epoch 106, Train loss 173.01042537391186, Test loss 66.78615540266037, Accuracy 89.49\n",
      "\n",
      "Epoch: 107\n",
      "Epoch 107, Train loss 177.96794403716922, Test loss 63.91496058553457, Accuracy 89.82\n",
      "\n",
      "Epoch: 108\n",
      "Epoch 108, Train loss 177.62039782851934, Test loss 64.53360660001636, Accuracy 89.85\n",
      "\n",
      "Epoch: 109\n",
      "Epoch 109, Train loss 174.7170418202877, Test loss 76.46712859161198, Accuracy 88.14\n",
      "\n",
      "Epoch: 110\n",
      "Epoch 110, Train loss 179.4522049240768, Test loss 65.59460578858852, Accuracy 89.55\n",
      "\n",
      "Epoch: 111\n",
      "Epoch 111, Train loss 176.1681998334825, Test loss 74.45628190413117, Accuracy 88.62\n",
      "\n",
      "Epoch: 112\n",
      "Epoch 112, Train loss 177.10667708143592, Test loss 66.10488952696323, Accuracy 89.6\n",
      "\n",
      "Epoch: 113\n",
      "Epoch 113, Train loss 173.33231996372342, Test loss 70.19649311900139, Accuracy 89.28\n",
      "\n",
      "Epoch: 114\n",
      "Epoch 114, Train loss 174.24574329331517, Test loss 64.8020853549242, Accuracy 89.91\n",
      "\n",
      "Epoch: 115\n",
      "Epoch 115, Train loss 176.0972884595394, Test loss 72.4169133901596, Accuracy 89.09\n",
      "\n",
      "Epoch: 116\n",
      "Epoch 116, Train loss 173.70244400575757, Test loss 71.48396686092019, Accuracy 88.59\n",
      "\n",
      "Epoch: 117\n",
      "Epoch 117, Train loss 167.99565632641315, Test loss 74.59661666303873, Accuracy 88.55\n",
      "\n",
      "Epoch: 118\n",
      "Epoch 118, Train loss 176.16264671832323, Test loss 64.74595981836319, Accuracy 89.86\n",
      "\n",
      "Epoch: 119\n",
      "Epoch 119, Train loss 175.44274279102683, Test loss 75.09280952438712, Accuracy 88.79\n",
      "\n",
      "Epoch: 120\n",
      "Epoch 120, Train loss 170.07440785318613, Test loss 79.75356091558933, Accuracy 87.63\n",
      "\n",
      "Epoch: 121\n",
      "Epoch 121, Train loss 170.7883945144713, Test loss 72.36373042315245, Accuracy 89.32\n",
      "\n",
      "Epoch: 122\n",
      "Epoch 122, Train loss 169.06301687285304, Test loss 65.80305968225002, Accuracy 90.02\n",
      "\n",
      "Epoch: 123\n",
      "Epoch 123, Train loss 171.46601279824972, Test loss 66.00133581459522, Accuracy 89.95\n",
      "\n",
      "Epoch: 124\n",
      "Epoch 124, Train loss 169.9736354276538, Test loss 70.1893615797162, Accuracy 89.3\n",
      "\n",
      "Epoch: 125\n",
      "Epoch 125, Train loss 169.65112160518765, Test loss 69.99682579934597, Accuracy 89.1\n",
      "\n",
      "Epoch: 126\n",
      "Epoch 126, Train loss 173.4031828045845, Test loss 64.91751128062606, Accuracy 89.69\n",
      "\n",
      "Epoch: 127\n",
      "Epoch 127, Train loss 167.30445004999638, Test loss 68.65072309598327, Accuracy 89.47\n",
      "\n",
      "Epoch: 128\n",
      "Epoch 128, Train loss 169.14390516653657, Test loss 64.15729764476418, Accuracy 89.93\n",
      "\n",
      "Epoch: 129\n",
      "Epoch 129, Train loss 169.68723759055138, Test loss 69.56516198813915, Accuracy 89.36\n",
      "\n",
      "Epoch: 130\n",
      "Epoch 130, Train loss 168.23556765541434, Test loss 71.87653625383973, Accuracy 89.06\n",
      "\n",
      "Epoch: 131\n",
      "Epoch 131, Train loss 167.4626528210938, Test loss 66.5296280682087, Accuracy 89.35\n",
      "\n",
      "Epoch: 132\n",
      "Epoch 132, Train loss 166.1876078993082, Test loss 73.19405720382929, Accuracy 88.84\n",
      "\n",
      "Epoch: 133\n",
      "Epoch 133, Train loss 165.05537458136678, Test loss 68.32122187688947, Accuracy 89.71\n",
      "\n",
      "Epoch: 134\n",
      "Epoch 134, Train loss 167.2959353812039, Test loss 73.94899694621563, Accuracy 88.78\n",
      "\n",
      "Epoch: 135\n",
      "Epoch 135, Train loss 166.43538294918835, Test loss 65.39862823486328, Accuracy 89.69\n",
      "\n",
      "Epoch: 136\n",
      "Epoch 136, Train loss 172.9600659608841, Test loss 68.08387256413698, Accuracy 88.97\n",
      "\n",
      "Epoch: 137\n",
      "Epoch 137, Train loss 167.45410224795341, Test loss 65.0721152164042, Accuracy 90.05\n",
      "\n",
      "Epoch: 138\n",
      "Epoch 138, Train loss 168.69933881238103, Test loss 64.87919044494629, Accuracy 89.98\n",
      "\n",
      "Epoch: 139\n",
      "Epoch 139, Train loss 166.84692188352346, Test loss 64.74736120551825, Accuracy 90.01\n",
      "\n",
      "Epoch: 140\n",
      "Epoch 140, Train loss 167.29254104942083, Test loss 66.48432983458042, Accuracy 89.83\n",
      "\n",
      "Epoch: 141\n",
      "Epoch 141, Train loss 166.41186259686947, Test loss 67.33736923709512, Accuracy 89.96\n",
      "\n",
      "Epoch: 142\n",
      "Epoch 142, Train loss 167.8367184922099, Test loss 64.77343009412289, Accuracy 89.97\n",
      "\n",
      "Epoch: 143\n",
      "Epoch 143, Train loss 163.17018181085587, Test loss 69.63355611637235, Accuracy 89.53\n",
      "\n",
      "Epoch: 144\n",
      "Epoch 144, Train loss 170.00313482806087, Test loss 69.8780310973525, Accuracy 89.18\n",
      "\n",
      "Epoch: 145\n",
      "Epoch 145, Train loss 165.59260676801205, Test loss 72.24000979214907, Accuracy 88.88\n",
      "\n",
      "Epoch: 146\n",
      "Epoch 146, Train loss 162.10591696947813, Test loss 67.44627219438553, Accuracy 89.64\n",
      "\n",
      "Epoch: 147\n",
      "Epoch 147, Train loss 171.08150872960687, Test loss 61.99004328995943, Accuracy 89.96\n",
      "\n",
      "Epoch: 148\n",
      "Epoch 148, Train loss 165.77455348148942, Test loss 73.42120287194848, Accuracy 88.55\n",
      "\n",
      "Epoch: 149\n",
      "Epoch 149, Train loss 166.6863021813333, Test loss 71.36952053010464, Accuracy 89.12\n"
     ]
    }
   ],
   "source": [
    "train_loss_history = []\n",
    "test_loss_history = []\n",
    "accuracy_history = []\n",
    "for epoch in range(0,150):\n",
    "\n",
    "    print('\\nEpoch: %d' % epoch)\n",
    "    net.train()\n",
    "    train_loss = 0\n",
    "    correct = 0\n",
    "    total = 0\n",
    "    for batch_idx, (inputs, targets) in enumerate(trainloader):\n",
    "        inputs, targets = inputs.to(device), targets.to(device)\n",
    "        optimizer.zero_grad()\n",
    "        outputs = net(inputs)\n",
    "        loss = criterion(outputs,targets)\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "\n",
    "        train_loss += loss.item()\n",
    "        _, predicted = outputs.max(1)\n",
    "        total += targets.size(0)\n",
    "        correct += predicted.eq(targets).sum().item()\n",
    "    acc1=100*correct/total\n",
    "   \n",
    "    global best_acc\n",
    "    net.eval()\n",
    "    test_loss = 0\n",
    "    correct1 = 0\n",
    "    total1 = 0\n",
    "    acc=0\n",
    "    with torch.no_grad():\n",
    "        for batch_idx, (inputs, targets) in enumerate(testloader):\n",
    "            inputs, targets = inputs.to(device), targets.to(device)\n",
    "            outputs = net(inputs)\n",
    "            loss = criterion(outputs, targets)\n",
    "            test_loss += loss.item()\n",
    "            _, predicted = outputs.max(1)\n",
    "            total1 += targets.size(0)\n",
    "            correct1 += predicted.eq(targets).sum().item()\n",
    "    acc = 100.*correct1/total1\n",
    "    train_loss_history.append(train_loss)\n",
    "    test_loss_history.append(test_loss)\n",
    "    accuracy_history.append(acc)\n",
    "    print('Epoch %s, Train loss %s, Test loss %s, Accuracy %s'%(epoch, train_loss, test_loss, acc))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "id": "oJ2xvlNeMQzj"
   },
   "outputs": [],
   "source": [
    "from matplotlib import pyplot as plt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "id": "dvKcDKFfR3Ca"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGD optimizer\n",
      "Accuracy History\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1461e03030a0>]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXAAAAD4CAYAAAD1jb0+AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAAyM0lEQVR4nO3deXhU5dnH8e+dfd9DEiAkQCDsa0AQXNgUrYpbW7W1aK3SVq1aa9XXt7b6tlWr1qW1Km617oobWMUF2REw7GEPAUIgywSyr7M87x8zGRISyAAJyST357pykTlzJnPnhPnNM/d5zjlijEEppZT38enoApRSSp0aDXCllPJSGuBKKeWlNMCVUspLaYArpZSX8juTTxYXF2dSU1PP5FMqpZTXW7duXbExJv7Y5Wc0wFNTU8nMzDyTT6mUUl5PRPa3tFxbKEop5aU0wJVSyktpgCullJfSAFdKKS+lAa6UUl7KowAXkTtEJEtEtorIna5lMSLytYjsdv0b3a6VKqWUaqLVABeRYcDNwHhgJHCJiKQB9wGLjDEDgEWu20oppc4QT+aBDwbWGGOqAURkKXAlMAs437XO68AS4N62L1EpdbKMMYhIq+sVV9YRGxrg0brtqc5mZ1X2YVLjQkmJCcHH58zVs27/EbbklTE4KYKY0AAOV9UTFxZIWo8w9zr1Ngf1dgeBfj74+3aezrMnAZ4F/EVEYoEa4GIgE0gwxuS71ikAEtqnRKUUQFWdjfyy2ibB0qDe5uDd73N5P/MAh0prsdkdPHbVCC4annTcn5dXUs3UJ5Zy0zl9uXfmIAD2WCrJLqqkrMbKlPQexIcHelTbW2v2U1Nv5xfn9Gvx/kOlNewuquS8gc0OJgTgmW92868lewCIDw/kgzkTSY0LPe7z5ZfVcO+HW7htShrj+8Z4VGNLjDHc9d4mco9UN7tv2qAejEqO4r9b8tlRUAFAQkQg791y/NqMMbyyYi+vrdzHe3Mm0Ds65JRr80SrbyXGmO3AY8BXwEJgI2A/Zh0DtHhlCBG5RUQyRSTTYrGcdsFKnYxV2cV8va3wtH+Oze7gP9/to9Zqb33l43jg4y3857t9p/z4vy3cwcXPLOfAMWGTXVTJjKeW8uCnW/H18WHmsET6xoXy67fX89aaFg/gA+CzzfnU2x08v2QPi3cW8ebq/cz4+1LmvLGO38/bzD+/3X3cxza+EMweSyV/mr+VR7/YQX5ZTYvrP/HlTm58bS1FFbXN7isqr+XVlXuZPjiBx64aTnWdjT//d3uTdWqtdl5dsZcDR6qptdr55RvrWLbLwl3vbaSqznbcOgFeX7WPNTmH3bcX7yxi3f4jAGzKKyP3SDUPXDyY124cx7PXjubNm87i7hkDWZdbwpNf7yIkwJc7pg3gvosGUW9z8PPXv6es2trseWx2B3/4NIs//3c7B0trWLS96IR1tQWPDqU3xrwCvAIgIn8F8oBCEUkyxuSLSBLQYrXGmLnAXICMjAy9/I86JU98uZPKOhvXndWHgQnhze7fdKCU6no7E/vHAs4R2p/mb+XLrYUE+PmQ9acLCfA79Y++y3ZbePDTrYQE+HH12N4n/XhLRR1vrcklKsSfH45NJjjA96Qeb7M73IH7zKLdPPHDke77/vHtbo5U1vPajeM4f2A8IkJNvZ1b317PAx9nMTgpgjF9ms8xWLDpEEN7RmB3GH715jpqrQ6mDerBXTMG8tCCrazPLXWv++RXOxmUGMEPRjhH9L94PZPKOhvP/WQMDy3YRqCfL9X1Nv69ah/3XzS4yfM4HIaluyw4DHy+OZ8bJvVtcv+z3+7GZjf84ZLBpMSGcriqnr8t3MmK3cVMHhDn/h2fW7yHRxfuYHBSBJvyyrhtShrPLcnmbwt38NCsYS1ut73FVfxx/lYGJ0Xw+W8mU1Vv57a31hMW5MfSe6awYNMhAnx9+NG4ZCKD/d2PmzwgjpvO6UtlnY0e4UHu5WP6RPOTl1fzyzfX8eoN45r8HV//bj9vrs5lzrn9+DwrnxXZxcw+O7WVv+zp8XQWSg/Xv31w9r/fBuYDs12rzAY+bY8CVdubu2wPn2482NFleKy0up7nlmTz71X7uOCpZdzyn0yKK+uarHPvh5uZ/dpatuSVUVpdz7VzV7NsVzEXDEmg3uZg66GyFn92ZZ2N5btb/2S4wRVmGw+UnNLvsHhnket3sZ7Stv8u5zCHq+oZnBTBR+vz2F3o/EhfXFnH51vyuWpsb6ak93D3soMDfHn22tEE+PmwYNOhZj8vx1LJ1kPlXDG6F8/9ZAwhAX5cd1YfXrx+LMN6RTK+bwzb88upqbdTVmPlucXZ/PXz7TgchuyiChbtKGLN3iNc+NQylu2y8NsZA7loeBJvr86lorbp6DTrUBmHq+rx9RHmH1PLvuIq3l17gGvH9yEl1tmW+PmkviTHBPPwZ1uprrexo6CcF5fmcPHwRC4alsimA6XcNX0gv7swnRvOTuX17/bz1dYCwPn3fOTz7azb7/w7vb5qHwDb88vZeKCUTzYcpKreTmF5HW98t5/PNh/ivPT4JuHdICTAr0l4A4zvG8Pfrh7B6r2Hmf3aWvfvWmu18+LSPUzsF8v9Fw9mclocq/ccxmZ3nNTf+WR5OiT5UES2AQuAW40xpcCjwAwR2Q1Md91WHWDZLgvV9cf/GLntUDmHXYGXX1bDYwt38r8fZ1FaXd+mddRa7Xy74/TbFcdatecwxsDc68dy94yBLNllYebTy/huj/NjcVF5LTsKKqi3OfjVW+v45ZvrOFRay5u/OIv/u9w5Mmt4QTfm7H9u5PpX1pJX0rwH2tjRAC9tsry63sZl/1zBo1/soN52/Bfrt9uLSIwIYlBiOP9etQ+Hw/DG6v186QqexhZtL+SN1U1bHws2HSIs0I9Xb8ggJMCPJ77aiTGG974/gNVuuH5iSrOfExbox3kD41mYVYDD0fTD72ebnbuvfjAiif7xYXz/wHT+esVw/Fw76EYnR2NzGLIOlbE65zAOAwdLa1i228IH6/Lw9RFe/lkGIjAoMZzrJ6Yw59x+VNTZeO/7A02ea8lOCyJw49mprM8tbdICmr/pEHZjuH1qmntZkL8vf/jBEHYVVnLe40u49a31RAT78+fLh/PMNaPJ/N/p3DF9AAD3XJjOsF4RzHlzHY98sZ3L/rmCF5flMOeNdewrrmLeujxmDEkgJMCXt9fk8ubq/QxJimBSWiyPf7mTwvI6Lh3Z87h/t5ZcMbo3T/94FOv3l3DdS2soLK/l/cwDFFXUcfs05+8xKS2OijobWw6WYbM7eG5xNpWttHpOhUcBbow5xxgzxBgz0hizyLXssDFmmjFmgDFmujHmSJtXp1pVUFbLz15dyz++zW7x/lqrnR++sIpfvbkeYwzvrD2Awxgq6228uCynTWt5eXkOP/93JlkHm452HQ7DjoJyTvUC2iuyiwkL9GPKoB7cPm0AC26bTFigHw98vAVjDMt3FwPwyJXDKSqvY3XOER65cjhjU6JJiAiid3Qw63ObB/gH6/Lc/fFjg7kxu8Ow8UApPgLb8yuoqT/aB9+SV8bmvDJeWLqHq19YxaHS5j3gepuD5bstTB3cgxsnpbKjoIJZz63kD59kcce7G5oE2oJNh7j5P5n84ZMsXly6x/34hVkFXDA0gaTIYH55Xj++3FrI3R9s4u01uUxKi6V/fPMdmwAXD08kv6yWjXlNf7/PNh9iXGo0SZHBAPgeM+tjVJ8oADbklrAyu5iQAF9iQwN4c/V+Plp/kCnpPZg+JIEl90zhg19OxN/XhxG9o5jQL4Z/LdlDYfnRXveSnUWM6BXpbics2Hx0FL4+t4QBPcLoEdF0pHvB0EQ+/NVEUmND2GOp4sFLhhATGgBAXNjRHashAX58MOdsLh3RkxeX5lBeY+WRK4dTUWvlin+tpLLOxq1T0rhsZE8+2nCQHQUV/HRCCr+7IJ16u4Ngf1+mD+7R4rY7kVmjejH3Z2PZY6nkkn+s4J/fZjMuNZqJ/ZwtvLP7O1s/K7OLmbs8h8e/3OkecLSlzjMfRp3Q/sNVjPjTl9z69nq255c3WQ7w4bq8Fj+urcwupqreztp9R5i/6RDvrM1lSnoPLhvZk3+v3Ieloq7ZY06Fw2F4PzMPoNl/1Ce+2snMp5ezMKv5aNMTK3YXM6FfrHv6VnpiOL8+P42c4irW55aybLeFuLAAfpyRzPM/HcOjVw7nqkZ96rEp0azbX9Jsx9vDC7YxPjWGAD8fNrkCvLLOxuur9jUJ6T2WSirrbFw4NBG7a1TaYIvrzeovVwwjx1LFHe9uwH7MaHft3iNU1duZmt6DWaN6ER3iz86CCu65MB1fER78NAtjDO9nHuDO9zaSkRLDD4Yn8cgXO3jq6138c3E25bU2Lh3hHCn++vw07po+kI/WH+RgaQ3XT2g++m4wdVAC/r7SZNsv2l7IrsLKE44848IC6RMTwobcUlZmFzO+bwxXZ/Tmm+1FWCrq+GGGc/uGBfoRHnS0/fB/s4ZRU293b4fS6no2HijlvPQeJMeEMKZPFPM3OgPcGMOG3NIW+/POv1sM78+ZyHf3T+Xy0b2OW2twgC/PXDOKl3+Wwee/OYdrx/fhwUuHUFJtZVRyFKOSo7jurD7YHYawQD9mjerJ6D7R/OSsPtw4KZWQgFM7q/bUQQl8/OtJhAT4OkffUwe4W1gxoQEMSYrgow0Hefqb3Vw8PJEZQ9p+op4GuJf4bs9hymttfLu9iIufXc4G14jyQIlzxFdUUceyFnq5X20tJDzQj0GJ4dwzbzOWijp+OqEPd04fSL3dwf0fbaaovOnMgE0HSt09RU+t2XuE3CPViMCavUcD/IPMA+7pYR+uP/neb+7hanKPVDM5LbbJ8otHJBHs78v73x9g+e5izhkQj4+PMG1wAteM79Nk3TF9oiksr+NgaQ31NufH2R88uxwfgSd/NJKhPZ07xQDeXL2fP87fyuzX1ro/8jZs659P7tvkNsDmvDKSIoP4yVkpPDxrKN/vK+HVFXubPP+iHYUE+vkwKS2OIH9f3r55Al/ceQ63Tknjtxeks3inhZlPL+f38zYzNiWaV28cx99/PJJzBsTxzKLdPLtoN3FhgUxKc47qfHyEO6YP4JXZGcyemMK0wccPhshgfyanxfH5lnyMMRSW13LPvM0MTorgRxnJJ9z2o5KjWLG7mD2WKianxXHtOOd2jQsLYOqglketAxLCeXjWUFbnHOGeeZt4dlE2DgPnpzunD146sic7CirYY6kkp7iKshoro12j/ZaIiPtTwomICNOHJLhH8teN78OfLh3CX68YDsCI3lHMGJLAzef0IzTQGdh/uWI4v3dNnzxV6YnhzL91Mm/edBbnuHa4Npg8II4cSxUhAb48dFnLO1lPlwa4l9hysIzwID8W/+58jIHv9zk7VnklztCMCQ1w9x7La60YY7A7DIt2FHL+oB48eOkQ6m0OekcHc97AHvSNC+V3F6SzZKeFcx9f7O65Wu0Ofv3Weua8uY4VrtaEJz7IPEB4oB+zRvZk7d4j2F1tk//5eAuT0mK54exUlu4qarXvnmOpxNrok8TKPc4aJh/z4ggL9OOi4YnMW5/Hkap6zh3Y9P7GxqY4R3jr9pdwz7xNPP7lTs4f2IOFd55LckwII3tHsSXP2av8dnsRsaEBrNtfwvWvrKGyzsaG3FIig/3JSIkmOSa4Sbtly8EyhveKBOCK0b24YEgCj3+1k+yiCvc6i3cUMbF/rHvGwuCkCHfLY/bEFEb2jqSoopa/XjGcd26eQFigH4F+vrx+43iW/34Ki+4+j6/uOrfZLJppgxN4aNawVg8suWh4EnklNdz0eiY3/yeTmno7/7xuNEH+J54JM7pPFBWuN7FJaXGkxoXyi8l9uWP6wBM+5w8zkvnZxBQ+Wn+QV1fuJT48kJG9owC4cGgiAAuzCtz7FY43Aj8dIsINk/oypGeEe9lLP8tw987bUmSIP5MHxDU7GGpKuvNN7o+XDvF4Pv3J0gD3ElsOljGsZySJkUHEhgawt9jZOjlwpIaE8CCuGtOLRduL+P28TYx5+Gvufn8TG3JLKK6sZ8aQBM7uH8c9F6bzx0uHuvudvzq/P4vuPo+MlBgeXrCV7KJKPtt8iIOlNUQF+3PnextbnLd7rPJaK59n5XPpqJ6clx5Pea1z5sBLy/bi7+vDP68dw9Vje2O1Gz7fcvyR/fuZB5j65FJecI3Ywdk+SYwIarHHe/XY3u52xeS0lg8QAedOtmB/X574aiefbjzEb2cM5IXrx9IzyjmyG5UcRY3Vzvf7Ssjcf4TrzurDc9eNYXNeGbe+tZ7M/SWM7hOFiDA6OdodPOW1VvYWVzGitzPARYS/XDEcfx/h1ZX7XH+favYdrub84xzA4ufrw3tzJvLd/dO47qw+TXrRPj5CckwI/ePD3P3fU3HZyJ7MnphCjqWSzXll/N/lw47bM29stCtYY0MDSHdN3fzfS4acsGXT4OFZw9j28IV8e/d5/Pf2ye7fq2dUMCOTo/hyawHrc0sID/TzqBZvNLF/LCvvm8oVo09+2qmnNMC9QL3NwY78CndQ9I0LJcfiDPC8kmqSY4L5UUYyNofh4w0HGZcaw0cbDnLHuxvx9xX3x9dbp6Q168OlxIby9DWjCPL35aEFW3l+yR7SE8J595aJVNZZ+cXrzp2SpdX1PPLFdua8kUlJVdNR9MKsAmqtDn6UkcxZfZ2tjs+35LNg0yGuGtOb6NAAhvaMoH98KJ8cZwrdR+vzuPfDzYBz1gk4++or9xQzKa356AZgQt9YkmOCGdoz4oQjHD9fH0YlR3HgSA3np8dz25S0JvePTI4C4NlFu3EYmDqoBzOHJfLny4exdJeF7KJKRic7w2xUchT5ZbUUlNW6d9YOd40uwXkU4aS0OJbutGCMYUV2y58gGgvy9211NHw6gvx9eWjWMJbcM4XtD8/0eB77kKQIAv18ODst7pQObQ8J8KNffPMdlDOHJrI5r4xvthUyqk/UGT1s/kzrFdV6++d0nNFrYiqoqbeTe6Sa9MTmB6Mcz67CCurtDob1OhrgS3c5+915JTWM7xvDgIRw3v7FWfSJDaFXVDD3friZ9zPzOGdAHBFBzee4NhYXFshvZwzkoQXbAHj6x6NITwzn6R+P4v6PtnDJP1YQGuBLtdWOn49wzdzVvHHTePcLc2V2setjciQiQnJMMC8uzcHmMMw+2zlaExEuH9WLJ7/exQtL95AWH8bZabEE+/vy6sp9/Pm/25jYL5aU2BA+2XAIq935plVabW3WW2zg4yO8OnucRwFw4dAELJV1/P1Ho5qtnxobQkSQH9/lHCYuLMD9cf/a8X04cKSafy3Zw7i+zgBv6Nd+s73QfQRgQwulwfnpPfhqWyHZRZWsyC4mISKw04wyT+YAogA/H167YRwpJzik/VTMHJbIYwt3UFRR12x/hTo5GuBn2G/f38iXWwv4+NeT3CO/1jTMdGgIir7xoXywLo/S6nryy2pIjna+y5+ddjTo/nLFcKJDA7hgSKJHz3H9hBTe+/4A1fV2LnEdbTdzWBJnp8Xx8rIc8kpruPmcfhypqufm/2RyzUur+fLOc/HzEVbtOczEfrHuUfJZfWOZt8755pHW4+gb1ZVje/P6d/t59IsdAIQH+jG8dySr9hxm5tBEnvrxKBbtKOSdtQfYdqjcPRI/+5gdmI0NaOGozJbcMKlvsyMAG4gII5OjWL67mCnpPZoE/D0XpvODEUkMSXL2Ukf2jmJcajSPf7mTIUkR9IoKbtbeaPjE8+2OIlZlFzNlUI8OP1nUqWr8f6qt9I0LZVBiODsKKk64A1O1TgP8DFq3v4QvsgoQgd/P28yC2yd7dHh3ww7MlFjniXH6uUZE3+1xHmDR0glz/H19mh3SfCJ+vj68ffMEbHaH+2AOgIggf357QXqTdf929Qhue3sDK7KLSY4OwVJR5z6EHeDs/s4Av3FSapPH9YoK5vsHplFWY2VbfjnzMvP4IquAOef2496Zg/DxETJSnCcmytzvnH+cnhDe7Gi49jCytzPAj53RISIM7Xl0hO3jIzxy5XAuemY53+Uc5qJhzd8ge0YFMzAhjNdW7qOk2srkdghBb3fJiCRyiqsY7eEgRrVMA/wMMcbw18+3Ex8eyIOXDOH2dzbw0IKtDOgRRl5JDXU2BxHBftw1fWCTAAXIcs10aBjF9Y1zfhxvmDbYO6Zt+mye7iibMSSBiCA/5m88xBjXDI+GAxjAudMsMTKoybIGIkJUSABn94/j7P5xPPFD02TEmxgZRK+oYFZlF7N23xGPdpi1hctG9SS7qPKEs1kapPVwzkN/ZtFuhveObHGd89N7MNd1oNQkDfBm5pzXn8tG9iIq5NR3zioN8Ha3fLeF974/QK3Vwbr9JTxy5XAuHdmTRdsLeWtNLgBB/j4E+PpQXmsjPTGCyxodYNGwA7PxaDYlNgQRWLbLuYMsuZ1PWXmsQD9fLh6exPxNhyitrqdnZJD70wE4R/MNR6K1pqX+9bjUaD5xHexxpkavAxPCeeH6sR6v/+sp/THGcOVxZhicPzCeuctyGJgQRkJE+3+C8Db+vj70iT2z/2+7Ig3wkzTnjUyGJEV6NJ/UZnfwwMdZlFTXExXiz7kD4/mhawbAo1eN4CcTUkiJCSE+PBBjYPrfl/LSshwuHZHkHm3vLGi6AxOcswp6RgZzsLQGH3GOWs+0WaN68e73B1i808KVY3q1aY93bGoMn2w8hL+vnNa5nttToJ9vs9ZSYxmpMcSEBjB1kJ4mX7UfDfCTtDrnCAVltccN8FdX7CWnuJKHLxvG51kF5B6p5oWfjmXmMb3SIH9fxqUeDScR+MU5/fifj7ewZu8RJrjaD6tcB7I0XhegX3woB0trSIoM7pArhIzvG0NiRBAF5bUttkpOR0ZKw4yPaPdRc94mwM+Hr+46l/Ag76xfeQedB34S6mzOU2vuKqxsdr4LcE4RfOrrXby5Openv9nFC0v20C8+lAs8PAfClWN6ERsawEuNTjK1dJeF9ITwZqPsvq4dmb2j23ee6fH4+giXjnTOVmm8A7MtDEwIJz0hvEkryRvFhQUS6Nd+87uV0uHBSThc6TyApcbqnMvd95j5sQu35lNRZ2NUchTPus4O+NhVwz0+UCHI35frJ6bw9De7yS6qICkymMx9JdxwzGwOOBrgyTEd10f8zbQBnJ/eo80vG+XrI3x517lt+jOV6op0BH4SGp+5b2dBebP73/8+j5TYEN69ZQIjekfSKyr4hGdRa8n1E1II9PPh5eV7WZ1zmHq7o8XrCHb0CBwgPMhfZ1go1YF0BH4SGgf4joIKZg5zthCMMeQeqea7nMP87oKBBPn7Mu+XZ1NTbz/pj9CxYYFcNbY389blUVFrI9jfl4zU5if7GZQYgb+vMDgpooWfopTqDjTAT0LDZbxCA3zZ6bpK9S3/yWTN3iPEhQXgI7jPQx3g53PK12C8aXJf3lmby3+35DN1UI8W3wQSI4NYdd804sJ0Hq1S3ZW2UE5Cwwj8rH6x7Cio4GBpDV9tKyQ1NgRjnGfH8+Tcxa3pHx/GdNcRgS21TxrEhwd67SHaSqnTpyPwk2CprCMy2J/hvSJZvLOId9c6D8T553Vj2nxn4m+mDmBvcRUXDNV5xEqplnl6Vfq7RGSriGSJyDsiEiQi/xaRvSKy0fU1qp1r7XDFlXXEhQUwOCkcY+CVFXsZnxrTLjNBhveO5JvfntcmI3qlVNfU6ghcRHoBvwGGGGNqROR94BrX3fcYY+a1Z4EdwRjDA59kcaSynpTYEK4d34fUuFAsFXXEhweSnujccVhdb+fKMSc3y0QppdqKpy0UPyBYRKxACHColfW9Wl5JDW+vySUuLICvthVQWm3lsatHUFxZz7BekfSJCSHI3weHcV6uSimlOkKrLRRjzEHgCSAXyAfKjDFfue7+i4hsFpGnRKTFS6KIyC0ikikimRZL84vudkaZ+53Xm3zjprOY0C+WHa4535YKZwvF10eYnBbHVWN6ERl84oslKKVUe2k1wEUkGpgF9AV6AqEi8lPgfmAQMA6IAe5t6fHGmLnGmAxjTEZ8/PFnVHQm3+8rITzIz3lId2I4uworqaqzUVlnc1+66+XZ49xXvFZKqY7gyU7M6cBeY4zFGGMFPgLONsbkG6c64DVgfHsW2l62HirjkS+2c+k/VrDSdf3CzH1HGJsSja+PMCgxnBqrnfW5JQDEhx39oKFT+JRSHcmTHnguMEFEQoAaYBqQKSJJxph8cabY5UBW+5XZPiwVdVzyjxX4ihDg58Mz3+xmaM8IdhVWMmuUc+dkww7LhovTxp3g4rlKKXUmtRrgxpg1IjIPWA/YgA3AXOALEYkHBNgI/LId62wXuUeqMQaev34s+4qr+Mvn23lz9X7g6ClNByaEIQIrdjsDvPEIXCmlOpJHs1CMMX8E/njM4qltX86ZVVBWCziv1TguNZonv97Js4uy8fcV9wWHQwL86BMTwtZDzh2Z8ToCV0p1Et36UPqCcmeAJ0YGERUSwBWje1FvdzC8VyRB/kfPP5LuuvK5iOfXjVRKqfbWrQO8sLyWAD8fokOcUwF/NjEVaH71m0GJzgCPCQnokKvfKKVUS7r1uVAKympJjAhyzyYZnBTBqzdkMLJ3VJP1GnZkxmn/WynViWiAH3PF8JYuQpvuGoFr/1sp1Zl0635AQXktCR5c0T01NoQAPx8997ZSqlPptiNwYwwF5bXM9CDA/Xx9+Mvlwxjg2pmplFKdQbcN8NJqK/U2BwkRrQc4wA8zktu5IqWUOjndtoWS75oDfmwPXCmlvEW3DfBC9xxw3TGplPJO3TbAjx7Eo1e8UUp5p+4b4GW1iEAPnRqolPJS3TrAY0MD9chKpZTX6rbpVVBeq/1vpZRX67YBXlheS2KE9r+VUt6r2wa4jsCVUt6uWwZ4rdVOabVV54ArpbxatwzwhoN4PD0KUymlOqNuGeA7CyoA9NwmSimv5lGAi8hdIrJVRLJE5B0RCRKRviKyRkSyReQ9Eel0p+orr7Xy7Y5CjDFNlm/LL8dHjl5pRymlvFGrAS4ivYDfABnGmGGAL3AN8BjwlDEmDSgBbmrPQk+Gw2F4Y/V+zn98CT//dybLXBckbrDtUDn94sMIDvA9zk9QSqnOz9MWih8QLCJ+QAiQj/OixvNc978OXN7m1Z2it9bm8odPskjrEUaAnw/Ld1ma3L89v5whSREdVJ1SSrWNVgPcGHMQeALIxRncZcA6oNQYY3Otlgf0aunxInKLiGSKSKbFYmlplTZVZ7Pzr8XZjE2J5r1bJpCREs2K7KMj8NLqeg6W1jCkpwa4Usq7edJCiQZmAX2BnkAoMNPTJzDGzDXGZBhjMuLj40+5UE99uO4g+WW1/GbaAESEyQPi2FFQQVGFc+bJtvxyAB2BK6W8nictlOnAXmOMxRhjBT4CJgFRrpYKQG/gYDvV6DGr3cG/lmQzMjmKcwfEAXBOmvNNY1X2YcDZ/wbnBYyVUsqbeRLgucAEEQkR5+XbpwHbgMXA1a51ZgOftk+Jnvt2RxF5JTXcPiXNfaX5IT0jiArxZ7lrR+a2/HJ6hAfqBYqVUl7Pkx74Gpw7K9cDW1yPmQvcC/xWRLKBWOCVdqzTIwWuA3RG94lyL/P1ESb1j2NFtgVjDNsOlWv/WynVJXh0TUxjzB+BPx6zOAcY3+YVnYaKWisA4UH+TZZPHhDHf7fk8+jCHeyxVDJ1UI+OKE8ppdpUlzoSs7zWRqCfDwF+TX+ti4cncc6AOF5aloPVbhjeK7KDKlRKqbbTpa5KX1FrbTb6BogM9ueNm86iuLKOzXmlnDdQR+BKKe/XpQK8vNZGRNDxf6W4sECmDko4gxUppVT76VItlIpaG+HBzUfgSinVFXWxALeecASulFJdSRcLcBvhGuBKqW6iiwW4lfBAbaEopbqHLhXg5TU6AldKdR9dJsCtdgc1VnuL0wiVUqor6jIBXlnrPLNtRLCOwJVS3UOXCfAKV4DrCFwp1V10mQAvd58HRUfgSqnuocsE+NERuAa4Uqp76DIB3jACj9AWilKqm+gyAa4jcKVUd9OFArzlc4ErpVRX1YUCXEfgSqnupQsFuJVgf1/8fbvMr6SUUifUZdJOT2SllOpuWk08EUkH3mu0qB/wIBAF3AxYXMv/xxjzeVsX6CkNcKVUd9Nq4hljdgKjAETEFzgIfAzcCDxljHmiPQv0VPlxLqemlFJd1cm2UKYBe4wx+9ujmNNRriNwpVQ3c7IBfg3wTqPbt4nIZhF5VUSiW3qAiNwiIpkikmmxWFpapU1U1FqJ0MupKaW6EY8DXEQCgMuAD1yLngf642yv5ANPtvQ4Y8xcY0yGMSYjPj7+9Ko9gYpWLmislFJdzcmMwC8C1htjCgGMMYXGGLsxxgG8BIxvjwI9VaE9cKVUN3MyAX4tjdonIpLU6L4rgKy2KupkWe0Oaq0OwgN1BK6U6j48SjwRCQVmAHMaLf6biIwCDLDvmPvOKD0KUynVHXmUeMaYKiD2mGXXt0tFp6C8Rs+DopTqfrrEkZg6AldKdUddJMBd5wLXaYRKqW6kSwR4uY7AlVLdUJcI8Aq9Go9SqhvqEgG+/3A1vj5CbFhAR5eilFJnTJcI8O9yDjO8VyQhAdpCUUp1H14f4NX1NjYdKGVi/9jWV1ZKqS7E6wN83f4SbA7DhH4a4Eqp7sXrA/y7PYfx8xEyUlo8GaJSSnVZXh/gq3MOM6J3JKF6HhSlVDfj1QFeVWdjc16Z9r+VUt2SVwd4pva/lVLdmFcH+OIdRQT4+TBW+99KqW7IawPc7jD8d0s+U9N76PxvpVS35LUBvnbvESwVdVwyMqn1lZVSqgvy2gBfsPkQIQG+TB3Uo6NLUUqpDuGVAW61O1iYVcC0wQnaPlFKdVteGeCr9hzmSFU9l47Q9olSqvtqNcBFJF1ENjb6KheRO0UkRkS+FpHdrn/P2FSQ7/cewUfg3IHxZ+oplVKq02k1wI0xO40xo4wxo4CxQDXwMXAfsMgYMwBY5Lp9RpRU1xMVEkCQv++ZekqllOp0TraFMg3YY4zZD8wCXnctfx24vA3rOqHSaitRIXrxBqVU93ayAX4N8I7r+wRjTL7r+wIgoaUHiMgtIpIpIpkWi+UUy2yqtKaeKL3+pVKqm/M4wEUkALgM+ODY+4wxBjAtPc4YM9cYk2GMyYiPb5uedUmVlegQvfqOUqp7O5kR+EXAemNMoet2oYgkAbj+LWrr4o6nrMZKpLZQlFLd3MkE+LUcbZ8AzAdmu76fDXzaVkW1pqS6XkfgSqluz6MAF5FQYAbwUaPFjwIzRGQ3MN11u93V2exU19u1B66U6vY8OozRGFMFxB6z7DDOWSlnVFm1FYCoUB2BK6W6N687ErO0xhXgOgJXSnVzXhfgJVX1ANoDV0p1e14X4O4RuM5CUUp1c94X4NXOEbgGuFKqu/PCAG8YgWsLRSnVvXldgJdUW/H3FUID9ERWSqnuzesCvKymnsjgAESko0tRSqkO5XUB7jwPiva/lVLK6wK8tKZed2AqpRTeGODVVt2BqZRSeGuA61GYSinlfQFeUl1PtJ4HRSmlvCvAa6126mwOInUErpRS3hXgJdV6HhSllGrgVQF+9ChMHYErpZRXBXiJngdFKaXcvCrA3RdzCNYWilJKeVWAl7gCPDpUR+BKKeXpNTGjRGSeiOwQke0iMlFE/iQiB0Vko+vr4vYutrTG1ULREbhSSnl2TUzgGWChMeZqEQkAQoALgaeMMU+0W3XHKKuxEuDrQ7CeiVAppVoPcBGJBM4FbgAwxtQD9R1xNsDaeruGt1JKuXjSQukLWIDXRGSDiLwsIqGu+24Tkc0i8qqIRLdfmU51NgeBfl7VtldKqXbjSRr6AWOA540xo4Eq4D7geaA/MArIB55s6cEicouIZIpIpsViOa1ia612gvx1BK6UUuBZgOcBecaYNa7b84AxxphCY4zdGOMAXgLGt/RgY8xcY0yGMSYjPj7+tIrVEbhSSh3VahoaYwqAAyKS7lo0DdgmIkmNVrsCyGqH+pqoszkI9NcAV0op8HwWyu3AW64ZKDnAjcCzIjIKMMA+YE57FNhYrdVOkJ+2UJRSCjwMcGPMRiDjmMXXt3k1raizOQjSEbhSSgFediSmjsCVUuoorwpw7YErpdRRXpWGdTY7gToCV0opwMsCvNaqPXCllGrgVWlYZ9URuFJKNfCqAK/VHrhSSrl5TRoaY6i3OXQErpRSLl4T4HU2B4AeSq+UUi5ek4Z1VmeA68mslFLKyXsC3GYHdASulFINvCYNtYWilFJNeU0a1lqdI3BtoSillJPXBLiOwJVSqimvSUMdgSulVFNeE+A6AldKqaa8Jg3ds1B0BK6UUoAXBXitex6415SslFLtymvS8Og8cB2BK6UUeFOA6whcKaWa8CgNRSRKROaJyA4R2S4iE0UkRkS+FpHdrn+j27PQhlkoOgJXSiknT4ezzwALjTGDgJHAduA+YJExZgCwyHW73egsFKWUaqrVNBSRSOBc4BUAY0y9MaYUmAW87lrtdeDy9inRqWEnpga4Uko5eZKGfQEL8JqIbBCRl0UkFEgwxuS71ikAElp6sIjcIiKZIpJpsVhOudA6mx0/H8HPVwNcKaXAswD3A8YAzxtjRgNVHNMuMcYYwLT0YGPMXGNMhjEmIz4+/pQLrbM5dPStlFKNeJKIeUCeMWaN6/Y8nIFeKCJJAK5/i9qnRKdaq10Po1dKqUZaDXBjTAFwQETSXYumAduA+cBs17LZwKftUqGLjsCVUqopPw/Xux14S0QCgBzgRpzh/76I3ATsB37UPiU66QhcKaWa8ijAjTEbgYwW7prWptWcQJ3NQYCOwJVSys1rErHO5tATWSmlVCNeE+C1VjtBOgJXSik3r0lEHYErpVRT3hPgOgJXSqkmvCYRdQSulFJNeU+AW+06D1wppRrxmkSstTn0XOBKKdWI1ySicwSuLRSllGrgPQGuh9IrpVQTXpGINrsDm8PoofRKKdWIVwS4Xo1HKaWa84pEbAhwHYErpdRRXhHgRy9o7BXlKqXUGeEViehuoeg0QqWUcvOKRGwYgQfpNEKllHLzigDXEbhSSjXnFYlY5+6B6whcKaUaeEWA17pnoXhFuUopdUZ4lIgisk9EtojIRhHJdC37k4gcdC3bKCIXt1eROgJXSqnmPL2oMcAUY0zxMcueMsY80ZYFtaROR+BKKdWMVyRirY7AlVKqGU8D3ABficg6Ebml0fLbRGSziLwqItEtPVBEbhGRTBHJtFgsp1SkHkqvlFLNeZqIk40xY4CLgFtF5FzgeaA/MArIB55s6YHGmLnGmAxjTEZ8fPwpFekegeuh9Eop5eZRgBtjDrr+LQI+BsYbYwqNMXZjjAN4CRjfXkXqCFwppZprNRFFJFREwhu+By4AskQkqdFqVwBZ7VOiBrhSSrXEk1koCcDHItKw/tvGmIUi8oaIjMLZH98HzGmvIhuuh+mqQSmlFB4EuDEmBxjZwvLr26WiFujVeJRSqjmvSMVBieHMHJbY0WUopVSncjIH8nSYa8b34ZrxfTq6DKWU6lS8YgSulFKqOQ1wpZTyUhrgSinlpTTAlVLKS2mAK6WUl9IAV0opL6UBrpRSXkoDXCmlvJQYY87ck4lYgP2n+PA44NgrAnU2WmPb0BpPX2evD7TGk5FijGl2Pu4zGuCnQ0QyjTEZHV3HiWiNbUNrPH2dvT7QGtuCtlCUUspLaYArpZSX8qYAn9vRBXhAa2wbWuPp6+z1gdZ42rymB66UUqopbxqBK6WUakQDXCmlvJRXBLiIzBSRnSKSLSL3dYJ6kkVksYhsE5GtInKHa3mMiHwtIrtd/0Z3glp9RWSDiHzmut1XRNa4tuV7IhLQwfVFicg8EdkhIttFZGJn244icpfr75wlIu+ISFBHb0cReVVEikQkq9GyFrebOD3rqnWziIzpwBofd/2tN4vIxyIS1ei++1017hSRCzuqxkb33S0iRkTiXLc7ZDueSKcPcBHxBZ4DLgKGANeKyJCOrQobcLcxZggwAbjVVdN9wCJjzABgket2R7sD2N7o9mPAU8aYNKAEuKlDqjrqGWChMWYQzmuvbqcTbUcR6QX8BsgwxgwDfIFr6Pjt+G9g5jHLjrfdLgIGuL5uAZ7vwBq/BoYZY0YAu4D7AVyvn2uAoa7H/Mv12u+IGhGRZOACILfR4o7ajsdnjOnUX8BE4MtGt+8H7u/ouo6p8VNgBrATSHItSwJ2dnBdvXG+kKcCnwGC86gyv5a2bQfUFwnsxbUzvdHyTrMdgV7AASAG5yUIPwMu7AzbEUgFslrbbsCLwLUtrXemazzmviuAt1zfN3ldA18CEzuqRmAezgHFPiCuo7fj8b46/Qicoy+gBnmuZZ2CiKQCo4E1QIIxJt91VwGQ0FF1uTwN/B5wuG7HAqXGGJvrdkdvy76ABXjN1eZ5WURC6UTb0RhzEHgC50gsHygD1tG5tmOD4223zvoa+jnwhev7TlOjiMwCDhpjNh1zV6epsYE3BHinJSJhwIfAncaY8sb3GedbdIfN0RSRS4AiY8y6jqrBA37AGOB5Y8xooIpj2iWdYDtGA7Nwvtn0BEJp4SN3Z9PR2601IvIAzlbkWx1dS2MiEgL8D/BgR9fiCW8I8INAcqPbvV3LOpSI+OMM77eMMR+5FheKSJLr/iSgqKPqAyYBl4nIPuBdnG2UZ4AoEfFzrdPR2zIPyDPGrHHdnocz0DvTdpwO7DXGWIwxVuAjnNu2M23HBsfbbp3qNSQiNwCXAD9xvdFA56mxP843602u105vYL2IJNJ5anTzhgD/Hhjg2usfgHNHx/yOLEhEBHgF2G6M+Xuju+YDs13fz8bZG+8Qxpj7jTG9jTGpOLfZt8aYnwCLgatdq3V0jQXAARFJdy2aBmyjE21HnK2TCSIS4vq7N9TYabZjI8fbbvOBn7lmUUwAyhq1Ws4oEZmJs613mTGmutFd84FrRCRQRPri3FG49kzXZ4zZYozpYYxJdb128oAxrv+rnWY7unVkA/4kdjJcjHOP9R7ggU5Qz2ScH083AxtdXxfj7DEvAnYD3wAxHV2rq97zgc9c3/fD+cLIBj4AAju4tlFApmtbfgJEd7btCDwE7ACygDeAwI7ejsA7OHvyVpwhc9PxthvOndfPuV4/W3DOqOmoGrNx9pEbXjcvNFr/AVeNO4GLOqrGY+7fx9GdmB2yHU/0pYfSK6WUl/KGFopSSqkWaIArpZSX0gBXSikvpQGulFJeSgNcKaW8lAa4Ukp5KQ1wpZTyUv8PjEiQx1TbrmAAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"SGD optimizer\")\n",
    "print(\"Accuracy History\")\n",
    "plt.plot([i for i in range(150)], [i for i in accuracy_history])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "id": "XrBu3nB9R6vm"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "SGD optimizer\n",
      "Test Loss History\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "[<matplotlib.lines.Line2D at 0x1461e02d3eb0>]"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXcAAAD4CAYAAAAXUaZHAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAAA2jklEQVR4nO3dd3xb9bn48c9jyXvvlWEncUL2MknYKSthFJrSUigXKONHB6XllraX0N7e3nu7oVC4bRkFyiiElbBXQgiEQEjiTGcPJ3HsOJ7xjoek7+8PHSnyip3YsWz5eb9efkX6niPp0Yn06Hue7/ecI8YYlFJKBZYgfweglFKq72lyV0qpAKTJXSmlApAmd6WUCkCa3JVSKgDZ/R0AQFJSksnKyvJ3GEopNaisX7++whiT3NmyAZHcs7KyyMvL83cYSik1qIjIwa6WaVlGKaUCkCZ3pZQKQJrclVIqAGlyV0qpAKTJXSmlApAmd6WUCkCa3JVSKgAN6uR+uPoYDy7dRUF5vb9DUUqpAWVQJ/eK+mYe+XgvBeUN/g5FKaUGlEGd3EPtNgCaHS4/R6KUUgPLIE/u7vCbHU4/R6KUUgPL4E7uwZ7krj13pZTyNbiTu6cs06o9d6WU8jXIk7s7/Ban9tyVUspXQCT35lZN7kop5WtQJ3e7LQhbkGjNXSml2uk2uYvIcBFZISLbRWSbiPzYav+1iBSLyCbr73KfxywUkb0isktE5p3ONxBqD9LZMkop1U5PrsTkAO4xxmwQkWhgvYgss5Y9ZIx5wHdlEZkAXAdMBDKAj0RkrDHmtGRgd3LXnrtSSvnqtudujCkxxmywbtcBO4DMEzzkauAlY0yzMWY/sBeY1RfBdibUbtOau1JKtXNSNXcRyQKmA2usph+KyBYReVpE4q22TOCQz8OK6OTHQETuEJE8EckrLy8/+cgtocFallFKqfZ6nNxFJApYDNxtjKkFHgVGA9OAEuDPJ/PCxpgnjDG5xpjc5OROL97dI1qWUUqpjnqU3EUkGHdif8EYswTAGFNqjHEaY1zAPzheeikGhvs8fJjVdlqE2m2a3JVSqp2ezJYR4ClghzHmQZ/2dJ/VFgBbrdtvAdeJSKiIZAM5wNq+C7ktnS2jlFId9WS2zDnAjUC+iGyy2u4DrheRaYABDgDfBTDGbBORV4DtuGfa3Hm6ZsqAVXPXAVWllGqj2+RujFkFSCeL3jvBY34L/LYXcfVYqN1G7TFHf7yUUkoNGoP6CFWAEJuWZZRSqr1Bn9zdUyG1LKOUUr4Gf3K3a81dKaXaC4DkbtOyjFJKtRMAyV3LMkop1d7gT+5ac1dKqQ4Gf3K323C6DA69GpNSSnkFQHLXi2QrpVR7mtyVUioADf7kHmwD0BkzSinlY/And71ItlJKdRAAyd3dc2/RAVWllPIKgOSuPXellGpv8Cf3YM+AqtbclVLKY/And7tnQFV77kop5REAyV177kop1V5PLrM3XERWiMh2EdkmIj+22u8XkZ0iskVEXheROKs9S0SOicgm6++x0/kGvGUZrbkrpZRXT3ruDuAeY8wEYA5wp4hMAJYBk4wxU4DdwEKfx+wzxkyz/r7X51H70LKMUkp11G1yN8aUGGM2WLfrgB1ApjFmqTHGc327L4Fhpy/MrmlZRimlOjqpmruIZAHTgTXtFt0KvO9zP1tENorIpyJyXhfPdYeI5IlIXnl5+cmE0YaefkAppTrqcXIXkShgMXC3MabWp/0XuEs3L1hNJcAIY8x04CfAiyIS0/75jDFPGGNyjTG5ycnJp/wGvKcf0Jq7Ukp59Si5i0gw7sT+gjFmiU/7d4ArgRuMMQbAGNNsjKm0bq8H9gFj+zhurxCblmWUUqq9nsyWEeApYIcx5kGf9vnAz4GrjDGNPu3JImKzbo8CcoCCvg7cI9gmiGhZRimlfNl7sM45wI1AvohsstruAx4BQoFl7vzPl9bMmPOB/xGRVsAFfM8YU9XXgXuIiF5qTyml2uk2uRtjVgHSyaL3ulh/Me4STr8JtdtobtWyjFJKeQz6I1RBL5KtlFLtBUZy14tkK6VUG4GR3O02nS2jlFI+AiS5B+k8d6WU8hE4yV3LMkop5RUgyV3LMkop5SswkrsOqCqlVBuBkdy15q6UUm0ESHLXsoxSSvkKkOSuZRmllPIVGMk9OIgWTe5KKeUVGMndbtOeu1JK+QiQ5B6kNXellPIRIMndRqvT4HQZf4eilFIDQmAk92D329C6u1JKuQVGcrfrpfaUUspXTy6zN1xEVojIdhHZJiI/ttoTRGSZiOyx/o232kVEHhGRvSKyRURmnO43EWq3LpKtPXellAJ61nN3APcYYyYAc4A7RWQCcC+w3BiTAyy37gNchvu6qTnAHcCjfR51OyGenrsepaqUUkAPkrsxpsQYs8G6XQfsADKBq4FnrdWeBb5m3b4aeM64fQnEiUh6XwfuS8sySinV1knV3EUkC5gOrAFSjTEl1qIjQKp1OxM45POwIqut/XPdISJ5IpJXXl5+snG3cTy5a89dKaXgJJK7iEThvvD13caYWt9lxhgDnNQ8RGPME8aYXGNMbnJy8sk8tIPQYE/NXXvuSikFPUzuIhKMO7G/YIxZYjWXesot1r9lVnsxMNzn4cOsttMmVGvuSinVRk9mywjwFLDDGPOgz6K3gJut2zcDb/q032TNmpkD1PiUb04LLcsopVRb9h6scw5wI5AvIpustvuAPwCviMhtwEHgWmvZe8DlwF6gEbilLwPujGcqZFOrlmWUUgp6kNyNMasA6WLxRZ2sb4A7exnXSYkOc7+NumZHf76sUkoNWAFxhGpCZAgAVQ0tfo5EKaUGhoBI7hEhNkLsQZrclVLKEhDJXURIjAyhsl6Tu1JKQYAkd3CXZo42anJXSikIsOReqWUZpZQCAii5J0aGUNXQ7O8wlFJqQAiY5B4fGUKV1tyVUgoIoOSeGBlCQ4tTD2RSSikCKLknRIYC6KCqUkoRUMndfSCTTodUSqkASu6JUXqUqlJKeQRMctdTECil1HGBk9wjNLkrpZRHwCT32PBgbEGiyV0ppQig5B4UJMRHBOtRqkopRc+uxPS0iJSJyFaftpdFZJP1d8BzEQ8RyRKRYz7LHjuNsXeQoEepKqUU0LMrMT0D/BV4ztNgjPmW57aI/Bmo8Vl/nzFmWh/Fd1LcyV177kop1W3P3RizEqjqbJl1fdVrgUV9HNcpSYwM1eSulFL0vuZ+HlBqjNnj05YtIhtF5FMROa+Xz39S4iODNbkrpRQ9K8ucyPW07bWXACOMMZUiMhN4Q0QmGmNq2z9QRO4A7gAYMWJEL8NwS4gMpfpYK06XwRbU1WVflVIq8J1yz11E7MDXgZc9bcaYZmNMpXV7PbAPGNvZ440xTxhjco0xucnJyacaRhuJkSEYo+eXUUqp3pRlLgZ2GmOKPA0ikiwiNuv2KCAHKOhdiD2nR6kqpZRbT6ZCLgJWA+NEpEhEbrMWXUfHgdTzgS3W1MjXgO8ZYzodjD0dEjW5K6UU0IOauzHm+i7av9NJ22Jgce/DOjUJevIwpZQCAugIVYDMuHBCbEHkHTjq71CUUsqvAiq5R4cFc9H4FN7aXIzD6fJ3OEop5TcBldwBFkzPpKK+hc/2VPg7FKWU8puAS+5zx6UQHxHMko3F/g5FKaX8JuCSe4g9iK9OzWDptiPUNrX6OxyllPKLgEvu4C7NNDtcfJB/xN+hKKWUXwRkcp82PI5RSZEs2VjU/cpKKRWAAjK5iwgLpmfyZUEVRUcb/R2OUkr1u4BM7gBfm54JwJubDvs5EqWU6n8Bm9yHJ0QwKzuBJRuKMMb4OxyllOpXAZvcAb4+PZN95Q1sKarpfmWllAogAZ3cL5uUDsDn+/SAJqXU0BLQyT02IpgQWxB1TQ5/h6KUUv0qoJM7QFSYnTo9mEkpNcQEfHKPDrNTrz13pdQQE/DJPSrUrmUZpdSQ05MrMT0tImUistWn7dciUiwim6y/y32WLRSRvSKyS0Tmna7Aeyo6zE5dsyZ3pdTQ0pOe+zPA/E7aHzLGTLP+3gMQkQm4L7830XrM3z3XVPWXqNBg7bkrpYacbpO7MWYl0NProF4NvGSMaTbG7Af2ArN6EV+vxYTZqW/WAVWl1NDSm5r7D0Vki1W2ibfaMoFDPusUWW0diMgdIpInInnl5eW9COPE3LNltOeulBpaTjW5PwqMBqYBJcCfT/YJjDFPGGNyjTG5ycnJpxhG9zyzZfQUBEqpoeSUkrsxptQY4zTGuIB/cLz0UgwM91l1mNXmN1GhwThchqZWvaaqUmroOKXkLiLpPncXAJ6ZNG8B14lIqIhkAznA2t6F2DvRYXYA6rTurpQaQuzdrSAii4C5QJKIFAH/BcwVkWmAAQ4A3wUwxmwTkVeA7YADuNMY4zwtkfeQN7k3OUiJ9mckSinVf7pN7saY6ztpfuoE6/8W+G1vgupLvsldKaWGiiFwhGowgJ6CQCk1pAR8cj/ec9eau1Jq6Aj45B4V6hlQ1Z67UmroCPjkHhPmLstozV0pNZQEfHKPDHWf2kZr7kqpoSTgk7vdFkREiE1r7kqpISXgkzu46+71WnNXSg0hQyK5R+vJw5RSQ8yQSO5RYcE6W0YpNaQMieQeoxfJVkoNMUMiuUeF6kWylVJDy5BI7lpzV0oNNUMiuUeFButsGaXUkDIkknt0mHsqpNOlV2NSSg0NQya5AzS0aO9dKTU0DKnkrnV3pdRQ0W1yF5GnRaRMRLb6tN0vIjtFZIuIvC4icVZ7logcE5FN1t9jpzH2HtNzuiulhpqe9NyfAea3a1sGTDLGTAF2Awt9lu0zxkyz/r7XN2H2jp7TXSk11HSb3I0xK4Gqdm1LjTGebvCXwLDTEFufiQrr+pzu93+4k+dXH+jniJRS6vTq9hqqPXAr8LLP/WwR2QjUAr80xnzW2YNE5A7gDoARI0b0QRhdi+mi5m6M4Z+fH6CxxUlosI1rc4ef1jiUUqq/9GpAVUR+ATiAF6ymEmCEMWY68BPgRRGJ6eyxxpgnjDG5xpjc5OTk3oTRra5q7uX1zTS2OIkOtbNwST4rdpWd1jiUUqq/nHJyF5HvAFcCNxhjDIAxptkYU2ndXg/sA8b2QZy90lXN/WBlIwB//MYUhseH89gn+/o9NqWUOh1OKbmLyHzg58BVxphGn/ZkEbFZt0cBOUBBXwTaGxEhNsKDbRyobGzTvr+iAYAJ6TGcl5PMtsO1uPRAJ6VUAOjJVMhFwGpgnIgUichtwF+BaGBZuymP5wNbRGQT8BrwPWNMVWfP259EhAvHp7Bs+xEcTpe3/WBlA/YgYVh8OJMzY6lvdrC/ssGPkSqlVN/odkDVGHN9J81PdbHuYmBxb4M6Ha6cnM67W0pYu7+Ks8ckAXCgopFh8eHYbUFMHhYLwNbiGkYnR/kzVKWU6rUhcYQqwNxxKUSE2Hh7S4m37UBlAyMTIwHISYki1B7ElqIaf4WolFJ9Zsgk9/AQGxePT+WDrSU4nC6MMRysbCQ7yZ3c7bYgJmTEkK/JXSkVAIZMcge4Yko6RxtbWV1QSUV9C/XNDkYmRniXT8mMZdvhGj17pFJq0BtSyf2CsclEh9pZsqGYg9bAaZbVcweYlBlLQ4uT/RX1/gpRKaX6xJBK7mHBNhbMyOTdLSVsKDwKQFbi8eQ+ZVgcgNbdlVKD3pBK7gA3zhlJi9PFo5/sw2ZNg/QYnRxJWHAQ+cWa3JVSg9uQS+45qdGcNSqRo42tDIsPJ9h2fBPYbUGMS4th15E6P0aolFK9N+SSO8BNZ40E8E6D9JUZF8aR2qb+DkkppfrUkEzul0xIZVRyJDNGxHVYlhIdRlltc/8HpZRSfagvTvk76NhtQSz79wuwBUmHZWmxYdQ3O6hvdhAVOiQ3j1IqAAzJnjvQaWIHSIsJA+BIjZZmlFKD15BN7l1JtZJ7qdbdlVKDmCb3dlJjQgHtuSulBjdN7u2kxVo99zpN7kqpwUuTezsRIXaiw+yUas9dKTWIaXLvRFqMznVXSg1uPUruIvK0iJSJyFaftgQRWSYie6x/4612EZFHRGSviGwRkRmnK/jTJTUmjCM9nOteXtfMF/sqTnNESil1cnrac38GmN+u7V5guTEmB1hu3Qe4DPe1U3OAO4BHex9m/0qNCaOshz33v63Yy3eeXqfXXlVKDSg9Su7GmJVA+2uhXg08a91+FviaT/tzxu1LIE5E0vsg1n6TFhtKWV1zj87rvr2klhani+pjrf0QmVJK9Uxvau6pxhjPNeuOAKnW7UzgkM96RVZbGyJyh4jkiUheeXl5L8Loe2kxYThdhsr6E5dmjDHsLKkF6HZdpZTqT30yoGqMMcBJ1SWMMU8YY3KNMbnJycl9EUafSfEcpepTmvnz0l189/m8NuuV1DRR2+QAoFyTu1JqAOlNci/1lFusf8us9mJguM96w6y2QaP9KQgcThf/+vIgS7eXUtt0vPyyw+q1A1TWt/RvkEopdQK9Se5vATdbt28G3vRpv8maNTMHqPEp3wwKxw9kcvfG1+yv4mhjK8bA+oNHvevt9Dnvu5ZllFIDSU+nQi4CVgPjRKRIRG4D/gBcIiJ7gIut+wDvAQXAXuAfwA/6POrTLCkqlCDBeyDTe/klhAfbCLYJa/cfH1feUVJLZlw4QQKVDdpzV0oNHD06p60x5vouFl3UyboGuLM3QfmbLUhIjg7lSG0TTpfhw21HuHB8Coerj7HOJ7nvPFLHhIwYmh0uKrQso5QaQPQI1S6kxYSx/uBRFq0tpKK+hcsnpTMrK4HNRdU0tTppanVSUF7P+LRokqJCtCyjlBpQNLl34Y7zR1NR18wv39hKWHAQc8clMys7gVanYWNhNXtK63EZOCM9hsSoEC3LKKUGFL3UUBeumJLO2aMTeXJVAUlRoUSG2skdmYAIrDtQ5R10HZ8eQ2JkKFuKqgH33PcWp4tQu82P0SulhjpN7icQHxnCz+ad4b0fGxHMuNRonlt9gGaHi8gQGyMSIkiMCvHW3N/YVMx/vbmNlT//CnERIf4KXSk1xGlZ5iRdOcV9JoX5E9P45y2zsAUJSVGh1Dc7aGp1su7AUWqbHHy6e2AddauUGlq0536SfnhhDj+8MKdNW1KUu4de2dDCnlL33PcVO8u4elqHsy70i33l9QgwKjnKL6+vlPI/7bn3gcRI96X5Kuqa2V1aD8Anu8t7dOKxU9HdGSjvXbyF+17PPy2vrZQaHDS594FEq+e+q7SOmmOt5I6Mp7qxlY2FR3G6DAcrG/rstUprm5j7wCf85p3tXa5TWNXI4Wq92IhSQ5km9z6QFOXuuX+5rxKA287Nxh4kLNtRyo8WbWTuA5+wtbim16/T1OrkjufXU1jVyJOr9rPuQPuzMEOLw0VZXTOltU24jyfzjxW7yqhr0tMgK+Uvmtz7gKfnvrrAndxnjownNyueJ1YW8G5+CQIs3lAEwLEWJ8+vPkCLw3XSr/Ofb2xl86FqHvrWVDLjwlm4JJ9mh7PNOu6kDs0OF7XHHL17Y6foYGUDt/xzHc98fqDPn/tfXx7kBy+s7/PnVSrQaHLvAxEhdsKDbZTUNBETZic5OpRLJ6RhDPx8/jjmTUzj7c2HcThdPPrpPv7zzW0s2156Uq9xuPoYr64v4o7zR7Fg+jB+s2ASe8vq+We7BFric2Hvsjr/lGa+sPZg1nayZ9Fbb24q5r38IxRXH+vz51YqkGhy7yOe3vvY1GhEhJvPzuK9H53HD+aOYcH0TCrqW3hj02Ge+qwAgFV7T26q5NJtRwD41pnusyl/ZVwKZ6RFtzmRGbh/BDxKe3gd2L622kruGwur+3RQ2eF0kW+Vt1b241TTz/aUU3USRyC7XMavJTGlQJN7n0m06u45qe7ph7YgYUJGDABzx6UQFxHMfa/nc6zVyYT0GD7bU3FSCeDDbaWMSYlitM/0xpGJERRWNbZZ73CNb3Lv/567MYbVBZVEh9qpb3aw80ht9w/qod2l9TS1ustZ/ZXcG5od3Pz0Wh5ZvqdH6xtjuPmfa7n75U2nNzCluqHJvY8kRbp77mNSojssC7EHceWUdFocLr4xcxjXzRpO0dFjHKxs7LBuZ442tLD2QBXzJqa2aR+ZGElhVWObqZGHq48RHuw+9UFZXf/33AsqGiiva+bms7OAtue/P5HfvLOdv63Ye8J1NluneJgzKoFVeytwOE9+3OJk7a9owGXg870VPVr/3fwSPttTQd6Brt/3ip1l/P69HX0VYkBpbHFw2zPr2FDYs8+N6pom9z7imTEzNrXzA4duOiuLmSPjufvisZw7JgmAz6yEcayl7aBo+1LGRztKcboM8yamtWkfnhDhnR3jUVLdRFZSJNGhdr/03D0lmWtmDiMtJox17ZLcB1tL2F/Rdmqo02V4cW0hD3+0x3v1q85sKaomNjyYG+dkUdfkYNOh6j6Pvz1PrHvK6rsdw2hqdfL793YCUFx9jKZWZ6frPfbpPh5fWcD2w323VxMoFq8vYvnOMl5ee6j7ldUJnXJyF5FxIrLJ569WRO4WkV+LSLFP++V9GfBA5am553TScwd3LX7x988mIy6c7KRIMuPCWbWnnNc3FjH51x/y9ubDgHumyZm//ahNL/bDbaVkxIYxOTO2zXOOTIjwPsbjcE0TGbFhJMeE+mVAdXVBJWkxYWQlRpCbFc96n0HVmsZWfvDCBv74/s42j9lbVk9ji5MWp4vHV+7r8rk3Haph6vA4zh2TRJDQL6d48P0h8vxwdeWJlQUUVx/jhtkjADrdM6travXuzby0rrAPIx38XC7D09YEgZV7ynXcopdOObkbY3YZY6YZY6YBM4FG4HVr8UOeZcaY9/ogzgHviinpfH/uaFJjQrtdV0Q4d0wSn+4u56evbsFpDH94fyfNDif3f7iLqoYW7v9wF+9sOcyitYV8uruMSyemISJtnmeEJ7n71N0PVx8jPS6M1Oiw0z6g2tTq5MFlu6ludA82GmNYU1DJWaMTERFyR8ZzuKbJO7Nl1d4KXMb9xfXt1XrKLTNHxvPimkLK65qpb3a02YNpbHGwu7SOacNiiY0IZvqI+H6pu++vaCA1JpTY8OBuSzNvbCrmvJwkrjtzhPXY+g7rfLGvEofLkJ0UyesbizvstflDWW0TH20vpbCHZUKAvWV1vL6xqE/jWLGrjP0VDZwzJpGSmib2lnXcfqrn+qoscxGwzxhzsI+eb9CZmBHLf8w/o0MC7sq5OUk0tbqYlBHDozfMpLj6GAuX5PPOlhK+e8EockfGc9eijSxcks+s7ATuunBMh+fIjA/HFiQcspJ7Q7ODmmOtZMSFk3qCnvsTK/dxw5Nfnvqbtby1+TCPLN/D86vd/+2bi2qoqG/h7NGJAORmJQCQZ/XeP93tvoZ6Y4uzTaLcUlRNVKidP31jCq1OF/P/spLJv/6QeX9ZyVFrlsq2w7U4XYapw+MAmJ2dwNbDtad0vMDJKKhoYExKFHNGJXineHpsOlTt/eFqdboorGxkyrBYspIivI9t75Nd5USF2vnfqydR1+Tg3Xz/Xl749mfzmPW75dz+XB4/fW1zjx/32KcF3PPK5i5LT6fiqVX7SY8N4/cLpgD9s2cWyPoquV8HLPK5/0MR2SIiT4tIfB+9RkCZNzGN/75qIs/cMov5k9I4d0wSSzYUkxgZwl0X5vD4jTM5d0wSv7h8PM/fOts7G8dXsC2IjLgw7+5/iTVTJiM2nJQYd8+9/a6tw+niyc/28/neyh5P79tRUsudL2zwPr/Hm5uKAXhtQxHGGF5aW0h4sI35k9xjA+PTY0iJDuXVPPfyT3eXc+mEVKJD7W3m+W8pqmFSZgyjk6P40UU5TB4Wyx3njaKwspHvPr+eZofTO0A5ZVgcAKOTo3C6TIfZQh5NrU4+29N9cqhvdnDPK5tZtLawQ6IyxrC/vJ7spEjOGZNE0dFj3t7tjpJarn1stXdgtLCqEYfLMCopiuiwYJKjQznQLrkbY1i5u5xzxiRyzphERiVFsmit/0ozVQ0tLN9ZylenZvD1GZmsP3iU2h4eVby7tA6XOX6R+ENVjTy3+kCPSykV9c389NXN3g7IoapGvthXyY1njWREYgRjUqL6LbkbY7o9X9Ng1OvkLiIhwFXAq1bTo8BoYBpQAvy5i8fdISJ5IpJXXj70fqFD7EHcfHYW8dYsm/+YfwZhwUH8bN44okLtJEaF8vxts/l/548iKKjrvYERCcenQ3rOJ5MeG0ZKdCgtDhc1x9p+WT/bU+EdgM3vwSkRmlqd3LVoI+/ml3DHc+u9ZYQjNU18sa+SnJQoDlY28snuct7afJivTk0nOiwYcE8Hve3cbFbtreDVvCJKa5u5eEIqF4xL9g4SNzuc7CipZaqVtO++eCzP3DKLhZeP54Frp7L2QBUz//cj/vjBToYnhJMc7f6RG5UcCdBhcNbjyc8KuPGptd3Ouvhw6xEWbyhi4ZJ8zvnDx6wpON47r2poobbJQXZSlHdv5I1NxTS1Ovn3lzfR4nSxzRoU3WeVEEanuAfUsxMjvbG9vtH9/Kv3VVJcfYwLxqYgInxtujuhHvXTVbw+31uBMXDrOVlcd+YInC7D53u6nxXkchl2W2c/3XbY/Rl6+vP9/OrNbV3+2PoyxvDL17fy2voi74+8pzR33phkAM7PSWbN/qrTUraqaWz1nr0V4PkvDzL798v7dC+kO29vPsz6g31/kJ+vvui5XwZsMMaUAhhjSo0xTmOMC/gHMKuzBxljnjDG5BpjcpOTk/sgjMFt8rBYNvznJVw3a8RJPW5EQqT3C+XtuceFkxrjvlJU++mQr60vIjbcnXx7cr6bP32wi71l9Xz3/FFsPVzDzxdvweUyvLW5GGPgL9dNIzLExs9e3Uxji7ND/N+ePYLoMDu/emsrAHPHJnPpxDQq6lvYdOgoO0vqaHUab4/c11VTM3jgm1O5ZEIqv7pyAs/dOtu7zHM644Lyzuuyb1kD1K/mnXjWxYfbjpAeG8aLt88mNjyYH/jsoXiS86ikSEYnRzFzZDwPLtvNBfevYOeROmZlJ3CgsoHGFgf7yq11rR+d7KRI9lc0YozhoWV7WLS2kG8/uQaAC8a5P+9nWmWrjYf8M+3vsz3lxITZmTIsjukj4ogOtfeot3zoaKP3eAPPjB/PntWags4TVkOzg79/spfdpXW8m1/CB9ZBeZutGU9bi2sJtglj09z/rxeMS6bF4WLN/hMPYveU7x7F/767nav++jnVjS0YY3hu9UHK65rZUtT78z+1t/5gFQ3NbU8D4nIZ7l28hW//Y81pHTfqi+R+PT4lGRFJ91m2ANjaB68xJESEnPzp9UckRFDV0EJdUyvF1U2IQFpsmDe5+06HrG5sYdn2Ur4+I5OsxAjyT/Bhrm1q5aFlu3n68/3cfNZIFl4+np/NG8fbmw/z7Se/5JW8IqYOj2NiRiyXT06nor6FcanRTLdq4h7RYcHcdNZImlpd7jJNTBhzxyUTbBOeXnXAO51xyrDYjkEA35g5jIe+NY1bz80mOynS2x4bHkxSVAgF5R177juP1LK7tJ64iGDe3lxCY0vn59hpbHF4S0Vnj0niiZtyaWp18v1/baDZ4fTWzLOTIhERXr5jDr9dMAnBvUdy+7nZGKs0sa+8nuToUGKsvZaspEgq6pvZUHiUwqpGvnfBaOaOS+Yr45LJjAsHYOrwWGxB0uNjATxW7Czjmke/4MIHPmHu/SvaHCjW4nDx3OoDXPjAJ3x0glNcGGNYtaeCs0cnYQsSgm1BnJvjHuTvrrTiKcXEhNnZdriWhmYH20vcMXxZ0HkyXrKxmD99sItLH1rJPa9sZuqwWM7LSWLzIfdncGtxDePSor2Xp5ydnUBYcBAf7yzr+YbphMPp4q5FG7np6bUYY3A4XXy0o5RjrU5ezSsiv7jGO3Db2Yn4euOVdYe45tHV/M/bbc/geuhoIw0tTkTg9ufyTluC71VyF5FI4BJgiU/zn0QkX0S2AF8B/r03r6FObGSie/CusKqRkupjJEeFEmwLIsUqX5TWNuN0GfKLarj/w120ON0HUk3KjPWWZY7UNPHB1iPe59x0qJrz/riCh5fv4bJJadx72XgAvn/BaP54zWS2Fteyt6yeBdMyALjWOiXC9bOGdzqgfMs52USG2LhkgvsgrJiwYH50YQ7v5pfwxw92khAZwrD48JN+76OSoijoZEbK25sPYwsSfr9gMvXNDt7PP9LJo91HuTY7XN7jB8akRPHna6ey6VA1f/14L/srGrAHiTc2uy2IG2aP5Mv7LuKXV4z3HoG8/XAt+8rrGZ18/MfH80P0xMoCaxtk8cwts/jnLcd3ZCNC7ExIj+k0udc1tfL0qv3c9PRa7n5pI797bwcPLt3FD15Yzy3PrONoQwsTMmI4XN3Eq3lF3sdc9vBKfvXmNg4dbeRvn3R9UFhBRQOHa5o4b2ySt+2CscmU1DSxp5tZKrut5H755HR2Hqll/UH3qa0TI0NYY50Oo9nhbJPoV+0pJz02jHsuGcv49Bge+OZUZoyIZ09ZHQ3NDvKLa9pM9Q0LtnF+TjLLtpd2+LHZWlzD8h2lbCmq7nDiPF/GGH75xlbe3nyYz/ZUsO1wLesOHKW6sZWoUDvPf3mQV/OKCLEHMTwh3Bt7X1i5u5yFr+cTag/ijU3Fbca3dpS4t99j/zaT0clR3pMK9rVeXYnJGNMAJLZru7FXEamT4pkOWVjZyMGqRjKsXmFKjCe5N/G9f6331jYvPCOFiRmxTM6M5Z0tJVQ1tPDfb2/j/a1HWH7PBYxOjuLxT/dhCxLe/uG5TPbpUYsI3zpzBGePTuKNjcV8M9ed1M/MSmDx989mWrteu0dSVCgrfjq3zTVl77ooB7stiD9+sJPZ2Qk9nmXkKzspkuU72/ZOjTG8vbmEs0cnMn9SGlmJEbySd4hrZg7r8PgPt5USFxHMrOwEb9v8SelcNTWDJ1YWcEZaNCMSI7DbOvaBRITMuHBiwuzsKKmloLzBewlGOF6eWbq9lKnD47x7Uu3NHBnPy+sO0ep0EWwL4lBVI//8/ACv5B2ivtnBmJQoCsrrqahvptnhIsQWxN0X5/CDuWMIsQdx6zPr+GDrEX55xXje3lzCvvIGHrl+OmW1Tfzm3R3sKKllfHpMh9f9zOotemrcAOePdd/+ZFcZY1M7P14D3NctGJ4QzplZCby07hCvri9CxP0D9sDS3RyqauTFtYU8+sk+3rjzHCZlxPDF3kqumJLOXRflcNdF7iuZTR3eiMvA+1uPUHOslUntjuOYNzGNpdtLyS+u8Zbtyuua+cZjX3jLQl+blsFfrpveaZxPrCzgpXWH+M7ZWSxaW8jL6w5htwkh9iB+deUEfr54Cy+uLWT+pDTiI4J5fUMxDqer0//vnmhscfCjRZvYdOgoFfUtnJEWzW8XTOaaR7/gpXWF/GCue8bbziO1iMDs7ERe+n9ziAi1ndLrdUePUB3kRlg99/s/3MXa/VWcZQ38RYTYiQ6z8+KaQpZtL+WHXxnD5/deyFM35wJ4e0kfbjvCh1b98+V1h6hubGH5jjK+Ni2zTWL3NTwhgrsuyiEy9HjfYObIeGwnGPhNiQkjxN724/b9uaN59tZZ/OKKCaf03kclR1JR39Jm0HjToWoKqxq5amoGIsI3c4ezZn8Vf/14T5vTFbQ6XSzfUcrF41M7fJnvvewMRNxTO0f5lILaExHOSI9h1d4Kao61tjnvz4iECETAGLh0QmqXzzFzZDzHWp3sLKljyYYiLrh/Bc+tPsBF41N4885z+OgnF7DqPy5k5/9eRsHvLmfH/8zn7ovHerfl/IlpFFcfY2txLS/nHWJsahRfnZLONTOGEWIP4sU1nc/GWbW3ghEJEd7PD7jHasamRnV7xtLdpXWMS4327rm8n1/CGWkxXGy9z/e3lvDsFwcAeCXvEJuLaqhrdnBuTlKb5/Ek7BfWuKfStj9I76LxKdiChKXbjsfz5GcFtDhcPHlTLl+blsG7+SVU1LvHlf70wU4e+9R9EFyLw8XjKwuYOy6Z//rqBOZPSuONTcUs3VbKuWOSWDAjk9SYUJwuwzUzMpmVnUhDi5PtJe4ptycz59/j9Y3FfLSjlPNzkvnZvHH86/bZzBwZzzljEnl+9UHv529nSR3ZiZGEh9iIjQgm+BR/TLqjyX2QiwkLJj4imIKKBm6YPYKfXjrOuywlOpTi6mPMGZXATy4ZS2ZcuLeHPNH6Iv3BOlp0xog4Fq8vYvGGYlqcLq6Z2T/Xf71gbDJjUk7tWq/tB1Ubmh3c9/pWosPsXGqVWr5zdhZXTknngaW7ueax1dQ0un8Ilm4rpbbJwfx2p3QAd5L73gWjAdrU+TszIT3GOxV1lE9ZJizYRkasey/qkm6SO8Cy7Uf477e3M2NEPKv+40Ievm66d06/h4h0mDl18YRUbEHCIx/vYfOhaq7NdZfG4iNDuGJyOm9sLO4w5tDU6uSLfZWc1y7ZAiyYPox1B462mU3iq8XhoqC8gXFp0YxJiSLEFoTDZcgdGc/YlGjiIoJ5YOlumlqdnJkVz9ubDrNseykicM7otq+XFBVKZlw4GwurCbYJ49La7i3ERYQwOzvB2/morG/mudUHuWpqBhdPSOWHF+bQ6jS8mlfEhsKj/P2TfTy4bDfldc18vLOUqoYWbj4ry73HmTucuiYHxdXHuGRCKsG2IH4wdwzj02M4LyeZWdbg9tr9Vdy3JJ/z71/RbS18b1k9T6zcd3xg9ouDTMyI4c/XTuXOr4zxnpLkO2dnU1LTxFLrR3PnkVrOSO96z6ivaHIPADfMHsnP5o3jN1+b1Kb3nB4bTkSIjfu/MbVDUogNDyYrMYKaY63Mm5jGXRflUNnQwp+X7uKMtGgmZnTeax9IPMm0oLwBl8twzyub2XWklkeun+6dERQZauev357B/10/nfyiah5ctguXy/DI8j2MTo7kK2ekdPrc3z1/NPMmpnp/JLoywafkMbrdBcnHpkaRnRRJzgl+vDLiwkmPDeOvK/bS0Ozg91+fTFps5yWcziREuhPgsu2lBNuEBdOP/yh/e/YI6podvL6xuM1jVu+rpLHF6e1p+7o2dxghtiBeWFOIMYaFS/K5+em1tFq9zoKKehwuw9jUaIJtQd7ZLblZ8QQFCbOyEmhxuLhqagY/uWQcdc0Onv58P5MyYr3Tfn1NHR5rbavjg6m+Lp2Qyp6yejYWHuX/Pt5Lk8PpvUC95+CyF9ce5Hfv7iAuIphWp4tnvzjAq3lFpMaEen/A5oxKZHhCOCLuPQKAm8/O4v0fn0ewLYi02DBGJETw+MoCXs47RFhwED9/bYu3M+DrYGUD//bkGi5+8FN+995O7nxxA5/vrWRXaZ33x8TXhWekkBkXzqK1hTQ0OzhY1cgZaR1LZX1Nk3sA+Om8cdz5lTEdPlS/uGI8z982m+EJEZ0+zlPjvOWcbM7PSSYjNozGFifXzOhYnx6IhsdHYAsS9lc08Oin+/hg2xHuu3w8XxnXMWF/dWoGN8weyfNfHuShj3azq7SOH12U02UpKTzExuM35nqnK3bFU88OtQd5Z8F4/O7rk3nmljO7HU+YMTIel4HbzxtFzglq3V25zDpo7OLxqW0OdssdGc/0EXE8/NGeNr33pduPEBli887d95UYFcplk9NYvL6IZ784YJ3+opyHP3Kf8niXNZjq6WVPTHd/hjxHI88dl0KILYi7LsphdnYCI6yT23W2lwB4j29oX5Lx8Py4Lvj7FzzzxQG+OiWjzZ7et2eP5FDVMfIOHuVn88Zx6YRUnl19gE92l/P1GcO8JbegIOHe+eP5/gWjSYnu/MdzVnYC5XXNnD06kUX/bw7l9c0sfH0L+UU1FB1t5FiLk7X7q/ja3z4nv7iGn80bx6+unMDneyv5/r/WExcRzFXWJANftiDhmpnDWLW3wpqNRKfjIH2tVwOqamDr7gP0b3NGkhkXzplZ8YgIN8wZySPL93D19I4f0IEoxB7EiIQIlm4/QkF5A1+dmsFt52Z3uf49l47lnS2H+b+P9zImJYorp/T+feakRmELErKTIjvsHaXH9mwG0NVTM6isb+ZHF3U8xURPXDY5nedWH+T289q+dxHhl1eM55pHV/PEygLuvngsLpdh2fYy5p6R0mlPGdx7gm9uOsyv397OWaMSyYwP52+f7CUxKoSl20qxBwmjktwJ9oY5I0iNCfX+sH3rzOFcMiHVe7DZN2cO48/Ldneot3t4Sk/tB1M9MuLCefi6aRxtaCE1JqzDnta8iakkRYUQFxHCt3KHMz49hg+tGv032w2iXzElnSumpNOVq6dlcKDCPSCdFBXKjy7M4aGPdvNeu9lWo5Iieeo7Z3pLdoVVjTzzxQG+e8EowoI736bfmDGMR5bv4U8fuMugZ6Sd/rKMDIQzr+Xm5pq8vDx/hzHkOV2GivrmLmd2DES3PbOO5TvLyIgN4/0fn09sRPAJ11+0tpCFS/L567en90lyB/jmY1+QkxrN7xZM7pPn62t3vrCBj3eW8cnP5lJ09BjXPPoFD183jaundT6uYozhsoc/o7S2ifd/fD7RYXau/L9V7K9oIDzYxvWzRvCrr/ZsELyxxcG7W0q4ZsawTo+0droM//ryINfMHEZU6Kn1NXeX1hERYmNYvHsP9d+sg8X+dfvsEz2sW8YY8otrKK1tpqqhmaqGVpwuFzfOyWrzOWt1unhjYzHzJ6V5j87uzLceX82a/VVEhdrJ//WlpzRDrD0RWW+Mye10mSZ3NZj94f2dPL5yHy/cPpuzR3feO2yvsLKxzSyR3mp2OAkSOW2zHnqrsLKRix/8lMz4cMakRLFiZxnr//MS77hEZ0prm3C4jLdHfrj6GPvK6zkzK6HL3ulA4TmZXPvZWf722voifvrqZnJHxvPa98/uk+c8UXIfWO9eqZP03fNH8dr3zupxYgf6NLEDhNptAzaxg/v9PnvrLFocLpZtL2XOqMQTJnaA1JiwNmMIGXHhnJeTPOATO7iT+kBL7ACXT07znu6hP2jPXakhoq6plX+sLOCCcSneKZiqfxUdbSQ+IqTNMSK9caKeuw6oKjVERIcF8xOf4yBU//OMC/SHgbfvopRSqtc0uSulVADS5K6UUgFIk7tSSgUgTe5KKRWANLkrpVQA0uSulFIBSJO7UkoFoAFxhKqIlAMHe/EUSUBFH4VzOgz0+EBj7CsaY9/QGHtmpDEmubMFAyK595aI5HV1CO5AMNDjA42xr2iMfUNj7D0tyyilVADS5K6UUgEoUJL7E/4OoBsDPT7QGPuKxtg3NMZeCoiau1JKqbYCpeeulFLKhyZ3pZQKQIM6uYvIfBHZJSJ7ReRef8cDICLDRWSFiGwXkW0i8mOrPUFElonIHutfv14KR0RsIrJRRN6x7meLyBprW74sIiH+jM+KKU5EXhORnSKyQ0TOGkjbUUT+3fo/3ioii0QkbCBsRxF5WkTKRGSrT1un203cHrHi3SIiM/wU3/3W//MWEXldROJ8li204tslIvNOd3xdxeiz7B4RMSKSZN3v923YE4M2uYuIDfgbcBkwAbheRHp2SfbTywHcY4yZAMwB7rTiuhdYbozJAZZb9/3px8AOn/t/BB4yxowBjgK3+SWqth4GPjDGnAFMxR3vgNiOIpIJ/AjINcZMAmzAdQyM7fgMML9dW1fb7TIgx/q7A3jUT/EtAyYZY6YAu4GFANZ35zpgovWYv1vffX/EiIgMBy4FCn2a/bENu2eMGZR/wFnAhz73FwIL/R1XJ3G+CVwC7ALSrbZ0YJcfYxqG+wt+IfAOILiPtLN3tm39FGMssB9r0N+nfUBsRyATOAQk4L5c5TvAvIGyHYEsYGt32w14HLi+s/X6M752yxYAL1i323yvgQ+Bs/yxDa2213B3NA4ASf7cht39DdqeO8e/XB5FVtuAISJZwHRgDZBqjCmxFh0BUv0VF/AX4OeAy7qfCFQbYxzW/YGwLbOBcuCfVvnoSRGJZIBsR2NMMfAA7h5cCVADrGfgbUePrrbbQPwe3Qq8b90eMPGJyNVAsTFmc7tFAyZGX4M5uQ9oIhIFLAbuNsbU+i4z7p93v8xBFZErgTJjzHp/vP5JsAMzgEeNMdOBBtqVYPy8HeOBq3H/CGUAkXSyGz8Q+XO7dUdEfoG7tPmCv2PxJSIRwH3Ar/wdS08N5uReDAz3uT/MavM7EQnGndhfMMYssZpLRSTdWp4OlPkpvHOAq0TkAPAS7tLMw0CciNitdQbCtiwCiowxa6z7r+FO9gNlO14M7DfGlBtjWoEluLftQNuOHl1ttwHzPRKR7wBXAjdYP0AwcOIbjfuHfLP13RkGbBCRNAZOjG0M5uS+DsixZieE4B50ecvPMSEiAjwF7DDGPOiz6C3gZuv2zbhr8f3OGLPQGDPMGJOFe5t9bIy5AVgBfMPf8XkYY44Ah0RknNV0EbCdAbIdcZdj5ohIhPV/7olvQG1HH11tt7eAm6wZH3OAGp/yTb8Rkfm4S4VXGWMafRa9BVwnIqEiko170HJtf8dnjMk3xqQYY7Ks704RMMP6nA6IbdiBv4v+vRzwuBz3yPo+4Bf+jseK6Vzcu7xbgE3W3+W469rLgT3AR0DCAIh1LvCOdXsU7i/NXuBVIHQAxDcNyLO25RtA/EDajsB/AzuBrcDzQOhA2I7AItzjAK24k9BtXW033IPpf7O+Q/m4Z//4I769uOvWnu/MYz7r/8KKbxdwmb+2YbvlBzg+oNrv27Anf3r6AaWUCkCDuSyjlFKqC5rclVIqAGlyV0qpAKTJXSmlApAmd6WUCkCa3JVSKgBpcldKqQD0/wE/F6w6nS0vAAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "print(\"SGD optimizer\")\n",
    "print(\"Test Loss History\")\n",
    "plt.plot([i for i in range(150)], [i for i in test_loss_history])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "id": "BhUzZ9kBkxs4"
   },
   "outputs": [],
   "source": [
    "model_path = './project1_model.pt'\n",
    "torch.save(net.state_dict(), model_path)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "JYj58B4Knivs"
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "collapsed_sections": [],
   "name": "acc93_(2).ipynb",
   "provenance": []
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.6"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "3e4fbcce4b2c4d3284eca06b7ea2c9a6": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_fcb0c93edfd64a0fafa6cc57a0e279f1",
      "placeholder": "​",
      "style": "IPY_MODEL_e8499808989a4bb8a71e8d8b45ad0683",
      "value": ""
     }
    },
    "418f9f3ec8a0421a89c33d8f6fea4b7e": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "ProgressStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "ProgressStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "bar_color": null,
      "description_width": ""
     }
    },
    "8d10a341d7ff44078bfece90053cfe98": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HTMLModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HTMLModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HTMLView",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_9abe1b6a23a341fba09ca1f2e8e106d2",
      "placeholder": "​",
      "style": "IPY_MODEL_ac7d242919984cef96d3a72a51916f92",
      "value": " 170499072/? [00:06&lt;00:00, 32917086.74it/s]"
     }
    },
    "8f660bd297b44104bf3db721bc818370": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "FloatProgressModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "FloatProgressModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "ProgressView",
      "bar_style": "success",
      "description": "",
      "description_tooltip": null,
      "layout": "IPY_MODEL_d16433ad0a68403c911ac4b2e42a73e4",
      "max": 170498071,
      "min": 0,
      "orientation": "horizontal",
      "style": "IPY_MODEL_418f9f3ec8a0421a89c33d8f6fea4b7e",
      "value": 170498071
     }
    },
    "9abe1b6a23a341fba09ca1f2e8e106d2": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "a327fc83f72c47a0a038163048052c43": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "ac7d242919984cef96d3a72a51916f92": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "d16433ad0a68403c911ac4b2e42a73e4": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    },
    "e8499808989a4bb8a71e8d8b45ad0683": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "DescriptionStyleModel",
     "state": {
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "DescriptionStyleModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "StyleView",
      "description_width": ""
     }
    },
    "f367da63ac8e4cdfa061246e17453d3d": {
     "model_module": "@jupyter-widgets/controls",
     "model_module_version": "1.5.0",
     "model_name": "HBoxModel",
     "state": {
      "_dom_classes": [],
      "_model_module": "@jupyter-widgets/controls",
      "_model_module_version": "1.5.0",
      "_model_name": "HBoxModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/controls",
      "_view_module_version": "1.5.0",
      "_view_name": "HBoxView",
      "box_style": "",
      "children": [
       "IPY_MODEL_3e4fbcce4b2c4d3284eca06b7ea2c9a6",
       "IPY_MODEL_8f660bd297b44104bf3db721bc818370",
       "IPY_MODEL_8d10a341d7ff44078bfece90053cfe98"
      ],
      "layout": "IPY_MODEL_a327fc83f72c47a0a038163048052c43"
     }
    },
    "fcb0c93edfd64a0fafa6cc57a0e279f1": {
     "model_module": "@jupyter-widgets/base",
     "model_module_version": "1.2.0",
     "model_name": "LayoutModel",
     "state": {
      "_model_module": "@jupyter-widgets/base",
      "_model_module_version": "1.2.0",
      "_model_name": "LayoutModel",
      "_view_count": null,
      "_view_module": "@jupyter-widgets/base",
      "_view_module_version": "1.2.0",
      "_view_name": "LayoutView",
      "align_content": null,
      "align_items": null,
      "align_self": null,
      "border": null,
      "bottom": null,
      "display": null,
      "flex": null,
      "flex_flow": null,
      "grid_area": null,
      "grid_auto_columns": null,
      "grid_auto_flow": null,
      "grid_auto_rows": null,
      "grid_column": null,
      "grid_gap": null,
      "grid_row": null,
      "grid_template_areas": null,
      "grid_template_columns": null,
      "grid_template_rows": null,
      "height": null,
      "justify_content": null,
      "justify_items": null,
      "left": null,
      "margin": null,
      "max_height": null,
      "max_width": null,
      "min_height": null,
      "min_width": null,
      "object_fit": null,
      "object_position": null,
      "order": null,
      "overflow": null,
      "overflow_x": null,
      "overflow_y": null,
      "padding": null,
      "right": null,
      "top": null,
      "visibility": null,
      "width": null
     }
    }
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}