train_cnn.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8eee28f4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from skimage import io\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "import os\n",
    "from glob import glob\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from tqdm import tqdm\n",
    "from PIL import Image\n",
    "from network import Net\n",
    "from torch.utils.data import DataLoader, Dataset\n",
    "from torch.autograd import Variable\n",
    "if not os.path.exists('./outputs'):\n",
    "    os.mkdir('./outputs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "745dbbdf",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Global parameters\n",
    "\n",
    "# If USE_CUDA is True, computations will be done using the GPU (may not work in all systems)\n",
    "# This will make the calculations happen faster\n",
    "USE_CUDA = torch.cuda.is_available()\n",
    "\n",
    "DATASET_PATH = './EuroSAT_data'\n",
    "\n",
    "BATCH_SIZE = 64 # Number of images that are used for calculating gradients at each step\n",
    "\n",
    "NUM_EPOCHS = 25 # Number of times we will go through all the training images. Do not go over 25\n",
    "\n",
    "LEARNING_RATE = 0.001 # Controls the step size\n",
    "MOMENTUM = 0.9 # Momentum for the gradient descent\n",
    "WEIGHT_DECAY = 0.0005"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "cc65618c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Dataloaders OK\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<torch.utils.data.dataloader.DataLoader at 0x154a4020f40>"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Create datasets and data loaders\n",
    "# Transformations\n",
    "\n",
    "from torchvision import datasets, models, transforms\n",
    "data_transforms = transforms.Compose([\n",
    "        transforms.Resize(64),\n",
    "        transforms.ToTensor(),\n",
    "        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])\n",
    "    ])\n",
    "\n",
    "train_dataset = datasets.ImageFolder(os.path.join(DATASET_PATH, 'train'), data_transforms)\n",
    "train_loader = DataLoader(train_dataset, BATCH_SIZE, shuffle=True, num_workers=2, pin_memory=True)\n",
    "\n",
    "\n",
    "test_dataset = datasets.ImageFolder(os.path.join(DATASET_PATH, 'test'), data_transforms)\n",
    "test_loader = DataLoader(test_dataset, BATCH_SIZE, shuffle=False, num_workers=2, pin_memory=True)\n",
    "\n",
    "class_names = train_dataset.classes\n",
    "\n",
    "print('Dataloaders OK')\n",
    "test_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "1b611aaf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Image label: 6\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<matplotlib.image.AxesImage at 0x154a3f92520>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAPsAAAD7CAYAAACscuKmAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjQuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/MnkTPAAAACXBIWXMAAAsTAAALEwEAmpwYAABE20lEQVR4nO29a4wk2XUe+J2IyHdlVdazu/oxL7L5GFGcIT2gaNBrUKRp0FrD/KWFBXjBXRCYP9qFjPXCJG1gARswTMOA4P2xWGCw1pqAtfYSlmUSsmCZOyvC8EKi2BKHnBnOo2em3696Z2XlOyLu/qjsvN85XVVdw+7OGivvBzQ6sm7kjRs3IjLOud853xHnHAICAv78IzrpAQQEBEwG4WEPCJgShIc9IGBKEB72gIApQXjYAwKmBOFhDwiYEjzUwy4iXxaRt0TkHRH5xqMaVEBAwKOH/Lw8u4jEAN4G8CUANwD8CMCvOed+9uiGFxAQ8KiQPMR3PwPgHefcewAgIv8awFcAHPqwL8zOurMrywAA+xOT5f4vwzRVbSm1ZS4fb+e52k31aX/E9Gc58DujHQ8a+uhrckjD8X8w6VSQ22PR5zjyRlcUHXbcA46tTtN/L461EZdlfvJ47gEg58/it+1ZCs2jHWEU8fEO7+Mo5HSBeT7iOD50P2duCp67Sqk03k5MH/1eb7zN99h+H3pfBp9PmmW+DzOORPVh5ttxC93rmZmto26D0bVut5rodbsH7vkwD/tZANfp8w0Av3TkF1aW8bv/9B8DAHIzgVut7nj71k5TtW13fNt2z2/T9QEADGl++8OBakvpBySO/UUf0gUC9I2Tm4sSS2G8LfQQ5NAX1tFUm/sGfXrIuoOhObYfY73ixzhTLqn9hI5nf9T4Hovohp6dm1H7NXf3xtu7e33V1uv7ceXxwT+0AFCg+YjNXNWqVRqjn+NhZOaKtsX8mHba7fH2XNmPvzHXUPu1O36//qCj2uolP8Zf+PBT4+2V+qza79Jbb4+3mwM9H5WaP3ZkzjOjH6G7zV3fR6ut9lus1/0H80PQp3dbn+6B3ba+waPEX097b977cf33v/PbOAwP47Mf9Otx3w+3iLwoIhdF5OLW7u4BXwkICJgEHubNfgPAefp8DsAtu5Nz7iUALwHAL374Q+4wW8SJ/91x5jcopza4hP6uzf3oUDNbvwHtG0SP47APAKJDzNEjbNP8KNPLIHo/Nu77xFHDiKz5wXB+7sX0wsO158lzzMZHbA6VKWvfuiTU6RGujL5mR72/6B4z3fH4+V4EgBzsUunvCbUJn6eYNy8P0ZxmTPMfkekeQ1udEZ1bZDsZHU+OcEMf5s3+IwAXRORpESkC+JsAvvcQ/QUEBDxG/NxvdudcKiL/A4A/ABAD+C3n3OuPbGQBAQGPFA9jxsM59/sAfv8RjSUgIOAx4qEe9veL3Dn0RivQmehDD3Lvn2zvtVTbFq0W7w29n55bmoVWK4+kZ+QIXzDzn7Nc+0yKWqHtKD7Kn9SeEq8rlBI9xoy84KN8Lz63zLAJTLFxm0v1XNWrFdpPH6vd9wupvS71X9DjLdBKfZ5pZqFSKI63ayW/3e7qVWpeSxlYLpU4qQGxKdbfZkfazke77b/XansmZ2VpUe3HawfdgV4L6gw9O5Qk+r4dDv15u8x/L0rNdaFTq5arqo3XN3iVPZM5tR8if+x2V7MOs/X9fUvFIg5DCJcNCJgShIc9IGBKMFkzPnfo9PZNndxwMGxlOsPjsJV5VHgvm0M26oy/l6beLcgsFURmYJoeHnCjqDwTIFQoFA78zmhg483EBOMwjZMTzTU0g3RkxxaSI6K7aFL5nAEd9JKRSQ8Ajo539e6GH5Nxa2IKWMlSfV0GAx/UxO5KraSPxWZrXNC3Y6vj3TlHwTIdE67RpyAgmGvWI/N/e3tnvF288Izaj6/Z0lxBtQmZz5a2LVNU3plTS9SirxlZ+Pfdw4mi9sg1iuxz4D/3h9rVqFZqAHSUoEV4swcETAnCwx4QMCUID3tAwJRgoj67A9AfhQYa9w+OQwFNWCYzWzZ7i5GQP5yb3dgni8mPTm34I4U5WvqO/TVFl9gMJ6JnrH/GySRDwyHx8bhP67NzJGZiaL9i7I9dpGQaOO3jFaiTM/M6KWShWh5v71I+w27HZB5lfrz1GZ1oUypwWLP/+9LygtqPk3qSgj6Xj9dPj7fLtF+3rWmn2+tb4+1KTdNVdyipitcH+qaPZUpUqVb0ucR0PWNDpZaIjjx/emW8bddSurTwtNftqraU5lVojJWK8b8Tfw+vN/dUU2+0ViFHxG6HN3tAwJQgPOwBAVOCCZvxDsOROWYZtGLBmyxFEwXkyORKWHjCUBOclWXFCcAWEbkMVkBiQByJjSxj01rg+7dRcsoVsOZcz9NEsZkDp+g2itAzqVYxuSS1io7GqpTJXWEq0lBBBXKHZor6NuAsrI894xMb371+R+23RTnxnZY+mX6R6DYyR7eahjdzNB8m7W95zpvWq4vL4+3q+bLa7/Sq7/PuttZCWNv19F2TTN+WGcfZ1TPj7YKhUnN2o4zLFov/XFXXXd+bzZY/9q0bN1Vbn8x6zkAUIyoCumZ3jRkvo4jFQV/rODDCmz0gYEoQHvaAgCnBRM14OOBePr6V1RlEZD5b+SNaheTkiNSY4LxCHtsIOkpwGZIogDX3Y4qW6kKbRGzWC8krJfeJYfj9IpOYUSazfl+zUx18vFnJvalaMJFlFVqBL5eMCU4uRZpykoZOVElpjMs1vfqckWl9bml+vF2v6/3evea1St67rk3T3tCPg5NFdo1aUUbSXDafaHvbJ83M0HnOs8QTgPl5P8aN5o5qa5PblPZ9FGGnp1fEB4PaeHvLmMhDMo1ZYgsAEjLjl+r+mlXimtrv8jWv4NZs62Nz5GCB7seyuT9m6Lxnlk6ptt5IzqpQ1NF/jPBmDwiYEoSHPSBgShAe9oCAKcFEfXYRGUeXRYau6hOlYWWgo5ij67yvmRgaJCdfP4b2d6IC9UHua27FH2hYFTNG7pKj3QrJ4dNoIwXLLLRgBAWHrI9P59YzmtlDOs9uX7eVKRONdaztfJf5XAq6jdcIdpubvrtYU6KnSQDCJL0hp/MUFRmo9+O0Q5frKL8+CTRst/x2biIP+3Se7a7O7uuTn57T2kFqahPwOkt9RvvbQrLTSWLmitaGIqKPSyaicH7JZ8SliRZn4boIvNYUiRU38W1dc57jKMXHJDgZEBDwXxDCwx4QMCWYrBkPb/akJqGlTWZaP9U0kTuszJChQThKLIn1qSWxN2854SKJbPSbb0tNQs72rqeC9oiOScUIJpCwQOQO719MlBWb6xy0ZS0zdnlKxqSVxLdRENt9um1sEqaG6owpgjEjM3iLxB8AICp4qqlW0VFtA7qGbMbvtDX1xmWRZme0sMWZpcZ4OyfX7v43FFGRBd2HRP6+GnY9pVYw5jjTdw1DMbaousugp+/NHlF47133VOTyUM/p+oZP1rlyU0cixhHfm/6+tXp3/MgUrFtW3p//3EbdEcKbPSBgShAe9oCAKUF42AMCpgQT142/V7k0N35ui7Ti7/NV+DepROGgtqQt+exlI7xXJIdnruypldhmg1FG2V5P0xs7lOWVDrx/nac6RJF94EKix8GVVZ3T/p9iWnhYJhyXfbzMOPQpiUwyPWgFQVhYs7WnhRwUrUh0Uq+3rfbrt70fPTDrJ5wVWJ/1fTgjxNHuEnWYaRqxTtl4TIdtbetxsMZkZs6zSsKaLOZRNGs6rW3vU9967x3VxuG+War7V7UK6DadW1xS+802/JrAam5CtGnto0gVe23YeEo0azXR91xl5Ovfl+3JYz20ZQQR+S0RWROR1+hvCyLyfRG5NPp//qg+AgICTh7HMeP/BYAvm799A8DLzrkLAF4efQ4ICPgA44FmvHPuP4nIU+bPXwHw+dH2twH8AMDXH9gXgOHI6syN+cwCDdWSpXEoEozMl8FQm33dHpmVfR0hNUvUULFGJZjiw8vl2Ggp1j9ncz810Xqs6250ECCRtznz7IisPS7jZEsqqzA0bVayGMdw6I8Vm3NJqY/mni7JNEOCGLtEiVYrmtbqklvDxwKApEwln8gdimJtfkaU7Wij2gaKgvXn2drTWWN9OrRlnuZm/HkXqFZBbqI052e9m1Axr0BiMyGGqo1Ya4+FJww1xl7OxpYW2Gh31v1+dN33etq96rZ2xtsXzp9TbedOL983Boufd4HulHPuNgCM/l95wP4BAQEnjMe+Gi8iL4rIRRG5yNI8AQEBk8XPuxp/V0RWnXO3RWQVwNphOzrnXgLwEgA888QTbruzb5rYiK4OBfYPTMXR4YArpvJquTbnugPfR9VEdPVi38bmYtlUJuVwtUpRr6TPNxq+PxrjntH9itWKuykDxAIYZpWdJaIzJTahf5NZC8/q0+Wk25xT/4OhXvlnCep0YEw/Mn37qT+3zCRmOPJRbDINRwOqCqymtBfPt41mzOm870WIAfdLd3fJhahUdBJLn867qGx8be+v0Op5srys2jr0kuoaOe3NHc8MsLvSGWrG4K0rPrruyu111ZaTyS/kKpbMPVylhJxCVUf5Xb69/wjaslCMn/fN/j0AXx1tfxXAd3/OfgICAiaE41Bv/wrAHwH4qIjcEJGvAfgWgC+JyCUAXxp9DggI+ADjOKvxv3ZI0xcf8VgCAgIeIyYaQZdmGdbuleMx/upwcDBlBEBJPAj7pJH1IUlU0pTuTTPv6/dJJKEWVc1+/nuZ8evKFNEVUTmlknH7WfPdGT83IkFLm23GhhbLzVtte0ZsVBo5m4tdYGdUNCKqj2VLA7OuviSeQts1FF2PSiXbstWHDdmuU/BuQ0NF9mktYcD3RGxvW/+9jhWvoOtUozJX9trukVCGM35viyjGbUP7cZTlYOC/1+5bGtEfb3ZWl9taXPQiIJWavx8rs9ov56WsK3f0Mtmld9/bP645f0aIjQ8ImBKEhz0gYEowYQ26CMkoscIyMML6Y8a6HZC5GMV+v1xs9BiXT9JmWmfgKaQtigrrZdrcSskss2WoymVvYpXJfJ4z5laPQrraRuygS+6FTU7hBCBOVJH88GhDS0M5RWUxBah2U65Blg9Nm5+ThGihWHQnTDEWTdRjxPNKlKitasveRWYi6LaI1iqT0IeNbGRYvb5S0e9bLPnr2eloc3yrSWWitnZU21Uq13R3y1RPzThqjoRJjFtTn/PpI0tP6FQS1snbplJnl+9qkYubdz1lt9XS4++M5m5gNBUZ4c0eEDAlCA97QMCUIDzsAQFTgon67JEIqiMBAWd+Z9Ih0zrad4vIV0yVr2l114k2c5a+I1+fNA/Lbe1bCS0YzGlXHLNUa+uZ82fH21ZsY0DUzfqu9vFurnsd9kFuw2X95yzlsFpTspmHbH1g8uE5dNbWvmNn2R0hbFEjXjHPdChqt+8nMjW15Aa0RpKR/2599iqtg0SR9vu7Le9Hc721lcaC2q/Z8qGoSjcfeq56qfeN2z19zru0VrNj1llubPlxrG1r+jFS9QP8dVpYbKj96qc8vbZhhDsvXbk23t7e9T57p6vXHzJaq4nNelJ9JNIRP4x4RUBAwJ8PhIc9IGBKMFEz3gEYjsKAcjk8esxmxMWUmcZlkUz1JBWdFpnoNC6lw5Rd36gdRPSRI8QAwJF5WychB0t/JSRmsWxoOaZZ7prSwCzPViBqLzN0ilD0mz1P1Ubmea2oTeRikWkiPQc10kFjKjIzoYJCrsDA0HJMQ5Upe5DNewCIyL2wunA97pPul9jo7deq/lrERnvQZQeLY2TG9epQ9Nv1tR3V1h4S7VfSEZdCYhNLy9696Of6PF99563x9o6JwuOsPaHxl4ypzsIZtaoex73nIooPf3+HN3tAwJQgPOwBAVOCiZrxWZ5jtzs4sC2lFWybQxFTNFKJzEMrdsB6ZmI7iQ5e6R6aFX1ejS9aMQXH2/6DNeNz6r9mJH9nSYSBK5MCWuSBJYGTyEhrU5uYtoRLIZEZaFepS2VKDOprszIjwYqlBc9AbO3o6qNRzIyBWWWvkWnNK8RGrIhnrt07XNJaswzGbSKTvG3UkIqkEbcw601fB+2idSjq8dqdLdV2Z4Mi+Yp6vufn5/wH0iW8fVsnquxlfk77JmqTE5EK5CqVI33N2G+tzWgTv723f95y39PjEd7sAQFTgvCwBwRMCcLDHhAwJZh4+afe4J6vZDKoyNe05ZYd+SrZ0B34d0CXFsqMX6dKIZEvnplyRCyqkZq2lMQLYxIZgNlPKJKqYPz+JaLieubQRfLhHUXNpcbHY//YRqSxsGSPaLNeV0d+lUnXPTfZZsU5Hyl3asmLL9qpYqRGJFSEyk9TBqLV8y8R1cSikgDQbxNtRpRluaB9Wab5Oh3t91fn/JoD32NZX0en9bt+3cJGofEawfzSomqbIb359677SLjcCHDWZ71vn+5q3fiYruHMDJWrMplz7bZfj0jMPTE3EqeMo0C9BQRMPcLDHhAwJZioGQ+nI5oYwqa7SWLhUkiOIstyS5uxeIDpg6PJVB6J+bnL5XA6ie1Yln6LjPZ8Qt+LjSkWl70Z3zA6ebttb0p2h2SCG00+dwTtx1FiXKIqsYkwpMUfmwg6rlrK2mxxYpNMSE+vZyPj/MSWiPabNbruHdJhdyZSkEU0uFRWydBfBZr/SllH0JWL/tgsOFIykXYNErk43dDRaXttTyNa3cCrt277NqLzFud1sg6XJrOUbkz0YETubcW4s+wK1Mz4z509AwD4A+MKMcKbPSBgShAe9oCAKUF42AMCpgST9dmxL2ABAM741EXyuyylViS/q8vZRAND45DYerkyp9qSIvnOLBJxn8iF95kKhtbi0NocTNEZ4Qb6XsH4yin1PzTrF0MSV+gPfJ+W8sqPCInkLKxcHUvvx8lRsdG27xM9trFJNcvEhO0SPViyNfN43YLXS4y/ymG8li7lbLk6hfdaaqxEYbsYmpDb3H/vybPnx9vnKAwYABYXPcXYWFlVba30lfH27U0dSsvrJ/WG14NnPxwAiuTrV2var+Z7gmOyG3WjnsIX0ZScltE6ixxRY+A45Z/Oi8gfisgbIvK6iPzG6O8LIvJ9Ebk0+n/+QX0FBAScHI5jxqcA/o5z7uMAPgvg10XkWQDfAPCyc+4CgJdHnwMCAj6gOE6tt9sAbo+2WyLyBoCzAL4C4POj3b4N4AcAvn5UXwIguWeyGCaoSqZ6wQgtsM5XkezPktUPpwwwa+plZEZxRNfQCFSwGd83ZY5jMjO5XLEYoXs27SJj+nJJ3bYpM8Slohxn8Bl9/JhMvftKNh9i4qcmSq6Xkca+Mf2ErsUOCWxIrKm3+RlvCmeiM+J4vockWDEw+m5lopAK5no6ispzREPduatLHi+RuTuo6jGWc+8affxprxv49Kouy9zs+DFuvHNdtbH2oK0lgBJl3KUUvZhpM7tMuvoD4771iJZjrf/ba/o8h0RTzpRM9t0oUtC6Qoz3tUAnIk8B+BSAHwI4NfohuPeDsPJ++goICJgsjv2wi8gMgN8B8Ledc7sP2p++96KIXBSRi+1O+8FfCAgIeCw41sMuIgXsP+i/7Zz7t6M/3xWR1VH7KoC1g77rnHvJOfeCc+6FWrV20C4BAQETwAN9dtl3GP85gDecc79JTd8D8FUA3xr9/90H9RVHEer3ssWM4CSHL1oBRK1B7umH+/TayRdq72orIlPsBunQ3+fjkmqIzTqi8Nbeuqdg7C+mrvOlW4dU084GDrO/zbr6sRERZKrPhsvGFPaZDn1b34TmxhxWa3zlnMJic1JLsSooccHP1RKVQ94fl99uUqnn3sDUlaMdZ0k4EgCGROcddg8AwDPnz4y3i+6MaisQBchKQ+s7WtHmraveT7/42huqrUMinosNTenmNK/rG97H3hvorDoVPlzRa1JcopxpVjfU15ZVjp48q73me/eIWd5ROA7P/jkA/y2AV0XkldHf/h72H/LviMjXAFwD8KvH6CsgIOCEcJzV+P+M+9bOx/jiox1OQEDA48KEdeMd8nvRccZ6blN2laWJVJYX/ewkRsSgUPZmYH+gD9CjiCM24+UIne1BqvvgiLpcUV5Ge56oOEuFDcif6Bvaj8+Nu08Mm8JZaZaWc2JFCvdhqciYxmVNP+XmkJltyyHnHOloO6E+FkjgYbfTV7u1297Ej2O9psOuzCaVQqoZM3hA890xdGZE43rl0lX/HRNS+LPLXnji8p1t1cZlujbb2vyfa3jar0/ulXU1QK6SFTktFTyd1yMBk3ig91tYWhpvnz+to/yu37gMAHCmb0aIjQ8ImBKEhz0gYEowcTO+d8/UMaoRHMB/X9VSSnBxJGiQGpOFxSziko50immlnqO7nI04onFZ7beUzDkuz2SZBRZhiEzUWZ+irLoDbdIyWPBBEm2CsyiFNc951ZcTbaL7SIfo0LYCjTnna2FcHhbmaO/pCLpKiZNYvNmdm9peOSUi9U1yB7tvvN1q62QX1nVv7ugQkMV5b/pe3/Dab7c3tKl+7fbGeLtjquvy6vmeKV9VoeXzlQUvWLHd3FH79YmFsQIepdjfq21ieYbmPM8sfmK8XauaZJrRvWQ1CRnhzR4QMCUID3tAwJQgPOwBAVOCyfrsDsjyQ35fOMTNiFeojLUjan4Vc+9DWn9eZaKxX2vojZR8SKspz8cbit+vZIQKOIAushlxFHGVmLUJjgiMiTKKTJhDgYQcbNZbSusFA6qrZ5cmCpRFFhnarKx8bD9vux1dE075jUako0g+e7PpfeWB8SmrBb9fbsQcMwp5KySHl33eaPr1gtzM6dqep8r4cnZN+GLGWYyGRczoelrxjf7Az0mt7oUqy6ZUN48xMyWb85IfGNOUsws6Wm91xUtGDEyeSTSWUT08hC682QMCpgThYQ8ImBJM3Iy3ZvM9cMSRjQoTMvEjMtOGJgoqy6iEz30lguiDciUON/ctraX2yw+mhQAgUiahPpeEKLvIlFtmk5wl6xOjX8+acdZo49JZKdF3dt7ZvchMkgzTkUyN5TY6jZOLjJugBBmIyqvEesQcDVgta7qU+8ioPNPS4pLar0hux8aO1ojrka4faN7afe0KcKmv1Mw367wnRmuPr2+PSko9dUZHuNWq3qzf3dVReEylLiz78lJnVnSpqWZ7x49/T/cx1tgP1FtAQEB42AMCpgThYQ8ImBJM1GcXEUX5KLBvaOiqAQs9UrjswOjGF0lnvGKELazv7L+j/UQhX9xmLikxC6L2ElOel6kse1heB7ArAnw8ptvE6LqzYEJsRTfpeDwHR13o2LQm5Iw3SF0oMesbMQlT5kY733FYLI1/b09TRoO+98WfWHpCtVWXvW9+as6HovZMeehbWzt+HGbtIKG6cJxFZ2lbFk/J76s1SLr3iQn3pe0+0542s41CjWfN/TJHdFu97jM3d9s6pHedBCgt5epG6wzO1ickhDd7QMCUIDzsAQFTgoma8VmeY3ssXqDNqLm6N2UqRU2bpaTB3e95KiUy0XisuZ0ZM4qpFabs7ssaU6aeNom4ZJXWv9P78bFzI4DB2X3WTWADMUo4wk3tprIArRtSsGWYRrClpizdxmgTzVUknTku1QQAHYqos9mDrRZTQ3SdTIRbY7bhx5TpuUoPEfp4892rar+MrqGYaEYu2czdx4a0ZF33YmQi+ch9cWaMQ7r223TO6buX1X5PUEbcxz70lGrje+ndq1d8fz0daVch+q5sSjYXR7qBcXyweAkQ3uwBAVOD8LAHBEwJJmrG94Yp3rq1LxJgE0QadZ+of2pR14hcbfjyOzMlv1rZMnLRAxI/cCbRQZvrtGpqVvR55dUu7XISDo/fSj1zFVrLPsQc4Zbq73EkFa8OR1bCOTk82cGROEafouZSW62WTm1oElDy1M9jkaZ4cWVB7bdL+nFV404snfHXjFfmM3PL7VGyzo/fvKba2mS6t8l948g6ADh1ypdy6hnBh8U5b/qyDuHyfEPtt7ZFK9+mdFNEbpMV3+BknQFN6l5f35tuzifJ8Oo7ANykclbX17yIxjDRpnon83Mwn2lzfSj797F1PRnhzR4QMCUID3tAwJQgPOwBAVOCifrsuQN6I79GrFBB0/tanY72u9Kej6Q6d8pvLy5oH5J1zTtG37tPWvRCNJQYH6wQUxkq6xuzz06RSiXjr3IyW24EMDjAaWj9aKLiUsowS60PSZ0kZk1AaezTsVJLa+HwPngVo0vz1u/qcSzO+7WVzGj9z9W8j7q25v3hyzduqv22Wp5eanaMjj5FiTniH0uVqtqPI/TmqLwyAJxaotLMJAy6u6vFPve2N8fbZ86fU21tivLb6Wvt/Naev1crZX/vnF8yZahiP9/rGzozb0gRgUw/rjf1fnHBr1e1C/oZuXex+6a8FuOBb3YRKYvIn4jIT0TkdRH5B6O/L4jI90Xk0uj/+Qf1FRAQcHI4jhnfB/AF59xzAJ4H8GUR+SyAbwB42Tl3AcDLo88BAQEfUByn1psDcM8mLoz+OQBfAfD50d+/DeAHAL5+VF8iQFSMR/3q3xlVjmeozefb214LvEO0yzmj0XXutDfZGlVNW+zs7Iy3y0RxDQ29xoISNnnmMBPJJj2kfRJCMOYtJ7E4E8XGdJvD4ZVaWaAgKusxxhSN5Y44Fhv1zrhUXI4ooT1bRoNuYdabzDYeb7NJ14wEKjb3tBncpjJdmUn4GZArU6HrOWM005n5XFrQgg8M1vyrGhGNT3/0Q+Ptz/zSp1XbT954dbz92uUrqq00503rJ0970926NU0WrCD9egA4vdAYb59fOTXe3uzoY6XEJ3dN4lF1VPrssIQv4Pj12eNRBdc1AN93zv0QwCnn3G0AGP2/ckQXAQEBJ4xjPezOucw59zyAcwA+IyKfeMBXxhCRF0Xkoohc5HTGgICAyeJ9UW/OuR3sm+tfBnBXRFYBYPT/2iHfeck594Jz7oUiRb8FBARMFg/02UVkGcDQObcjIhUAfwXAPwHwPQBfBfCt0f/ffVBfDj47yubYczJ+2VArCVFbKWUk9QydxCKCxUj7f4vL3hfqUY21G7duqf24//t9ce8zORaoMMXSWLN+mBmfmnzgclGvKyQU0ssU2sD47E6JV+hLOEPZWyll3A2Nz56Sn15IjA+shDP83yOTfpdR/wUz373U++YzDU/UlKs7ar/2kKw9Q4My5Vil9YxaTYebZuS/Nrs6THWj6Sm1s8t+TefMoqZtF2j9p1TUc1ohIczlOa0HX5/3/SSRXwv66WtvqP0GzvfZMmKXjYo/t488QwIepl7h3R3v6/f7mjpsjMaVWDETwnF49lUA35Z9uZQIwHecc78nIn8E4Dsi8jUA1wD86jH6CggIOCEcZzX+pwA+dcDfNwF88XEMKiAg4NFjohF0cRRhdmSC2TLBTPdYQYbh0JssbADdNOV5tygLKzOm0jKZaWdXfBRepazXEXYoa2poxsjBcJzpZsmOPGda5HCdcQvOPuP9MkONcaZeDn2exYKeu3uw2vYsmFCyggesna/KZWtXoL3rSxo1FnRMVbHgXQOmNxMjgJGDSj0bOpZN0mqNXA2jz7FD5ZQ22vqeKNL4I5qbWkVf93TLn/P1jXXVtr5By1FmHudrnn7caPr7b3egr9mQojEzo+E/IPewnvgxfuIJHck3S0IibRPJt7C0707YUmSMEBsfEDAlCA97QMCUYLLln3KH4b1kCrMcz5Fl3a7m43n1nE07Z8oRzc5QRJcx4/du3B5vp+QWPGmSHkCrvtsUdbc/SG8Sag06U002Id0zE/ymkl1MlJXQinaukm7Mij6t2ttV2W7Pj6tAJl3RlMPixJLBUEdjlWhfZklSE0HIJnJq+ujQNWyT21GuaDejTCvdPVNalctG8XWvVDVbI6TV1tnVCSKO2I9NqiYbGddotkoSzl1tIjMrMFvWDMoeJ8JUPEtQNVF+620/xorRCVyc9y7mCrlDO3sttV+RHMb507q8VKm2vxp/VMmy8GYPCJgShIc9IGBKEB72gIApwYTFKxy6Y7/P0Enx4VruXKK4QPRJZqgaTn8ql7XPBIqyyo/QMa9TuaNh1/jD8L5cQscaGt+biZXY9O/ovC0dxgF7CfnK95UqIp+vZPw/LhvF4oO2XBBHB9qMtTijtQmK/BLDebG/PTOj/ejWwM8Vz2Mp0WsHC3Oz4+31/oZqmydxkkUSId01ORZKaNSIfwrpxscUkWaqfSOF36+Xar8/z/2cds0izJ01LzAxO+/n+8I5LV7xBK0TnTWlmE8v+c8cATgwVKfQWsrNDS3OcuP1KwCAvfbh+SfhzR4QMCUID3tAwJRg4lVcvYluKAKO2jImZ4E+s5nqjACBEJ1Xm9VaZEWimhIyh96+rLXKi5QEwZQIABTIJGxR9JiNiuOkkMwkd7A5fV/5J+qHEz96hl7Lcbh5zn1YN0H1oag9fRuwWSx8rPs08/hYui1JDr61Bj19LnOU1BIv6D7KNYpyI6ps11BSXdLKz02AIlea5TpaNjqyPfDnbK/LkGhF1jkEgCZpwW0TDVevaTeyPuM/91p6/H92/cZ4e4n07C9c+JDar0c6+u9e1VF+l6/vU8tdQzkzwps9IGBKEB72gIApQXjYAwKmBBP12fflK/KDm9hPMnXauNZWtex9sMrsrNqP/cTMlCjeJkpC1R4baB+SQ1OtwOLTq14AY578+d2WpkGYirN12dgbjIzAIuua5+QDM8UFaL/sPjFKop5SmtNIDv9dF+NvdylcNC/7trrJWCsSRVozVGdOx9va3KEGfV3m614MYrWhhSG2m9635YxGZ86FS2QPjL/NpaQdrW9IwWYjeid+2dQj6HS9L75OIbeAFkrtp/5+YcFNQGv9N0wo7Wmq/VbjkG+bT0n6/pEJf+6MxnFEqbfwZg8ImBaEhz0gYEowWerNOcTGjLuHIpmqszVtzs03vLleItOxZxL496hsVG7KHOvyR2TGG9rJkbnVNCWhb8ae7niGsuUapnzS9rYvd9Q37oSiyozJxVQZJwXaUllG9V21MPXGUXixiVwTcjVsWWnWV4+VppnR0SdKamj0+lgnjs/LRkf2B970nZnR2nJVini7Sy5Vz5Sh6tE4EiPewBp91aq/r3JTqpvHO2fMbB6jFVZhZtVRdB27noAuHc1ltQFA6B6MSUxlt6Uj+SpV/xxIrOm7ZKRn+NC68QEBAf/lIzzsAQFTgoma8cVCgqdOH1yepz7jV7dtYsmQIrq22n36uy3HRMkjR0SPMazZo+SSjcm5TWbV9TvepD+92FD7LVNiw/rGpmrrU6QWzMoxR3hFxFrEZtW+QOOy448pei+m/pwx9x2Z3an1J6jLIZm7PbNq3+lS9dHIJLhQSSMefiq6j43mjv9gXJ4koXJYFBpnz5mjDa3DMyS9t0Hs+8962kVbbDzl9zNRfk1iBTiKEtAJVzPEVszOaFfgyRUvY33zho7a5PuK5el2NnUV11aPyj+ZSLneSPzlqNs+vNkDAqYE4WEPCJgShIc9IGBKMFGfPUkSLI6ik6zf1e54Gq1jBCe7JHTIvrjYCKPDWYfDx2Qy5zjjyWZu8RrB2pan1zptHVX17Ec+Mt5emNOZc22KBLP0YIHosQL9DGe23DKV7rUu2oBoHV77sJF2/DmxtNwhmXOJoehAfbSMaEKJ/NcylaTa7RuRTS4vZUoXOY4opNjDgokeq9JkdY3gSDqkMabe914wwpf1mhffuLmjr6eaO3MtchJFGVI03UxVR3eeO+ujL/eaeh1nY9v75n0S5miZ7Litlm/LjNhqeSTq8kiot1HZ5h+LyO+NPi+IyPdF5NLo//kH9REQEHByeD9m/G8A4Gp13wDwsnPuAoCXR58DAgI+oDiWGS8i5wD81wD+EYD/afTnrwD4/Gj729gv5fz1o/oZphlubzYPbBPiZ3qpNjkzFrpgs/I+4YajIsv8Nn/NarIXFPOm+y9SdBab+M5EBd647csFnVtqqLblc6u0313VNiTqaUgVUlND0fXJrLTmuYrQo3MbWoOfaDkxAhsFpV/vj23pTCYEi8bEz7nCK0XkFYyvlXEnJgqvSJFsJYqw3N7VSSZDGldmMkFU6Sk69DMf1rrrEWnD393Q1yWia100rkaZ7gm+Ts2Wdmsu3/R1C1ptTfstU4TomRWfhCMm0m5z1ydczZQN1TnSAEziw9/fx32z/zMAfxeaxjzlnLsNAKP/V47ZV0BAwAnggQ+7iPx1AGvOuT/9eQ4gIi+KyEURudjtdh78hYCAgMeC45jxnwPwN0TkVwCUAcyKyL8EcFdEVp1zt0VkFcDaQV92zr0E4CUAWDl15nhhbQEBAY8cx6nP/k0A3wQAEfk8gP/ZOfe3ROSfAvgqgG+N/v/ug/oaDoe4e2vfH7LCgOw723BZRoH9J+OfsL9q6RluEwo3dabmHI9jrq7pE1WnjfxLG7XbJDGLkum/RKGdscmMalMocMbzYcJIM/KHrc+exOQr0xwUY0018ffyoe7D0RzzXEV2rmj+Z+o6U7FAtFmj7gUZuqZccZfFQ6ywRaMx3r5DPnBqx8sipKkeY5r5/mcorPmJ82fVfk0K27V12hytR4i5FlzXr09rB2sbWgNfBt6qjTMd6nqOdORPk4BHYtY3dlve19+5pt+t87X9cXCItMXDBNV8C8CXROQSgC+NPgcEBHxA8b6CapxzP8D+qjucc5sAvvjohxQQEPA4MNkIOhE0RpSBs9YGmdkzM1oDjNmOMmUdFQu6fG6RSkNZzfShKgntTbvYiB1EZAbvtTRNyNp1bEgOUitQ4fdbMwIYcWVnvL2yqDMAd9uU/cTZcSbrjemk1ER0sRBCkea4YDL42E2AiSJkug1Ew1mBioT6LBhNN0eRciyAUTFZY6x/d8ro9Jfo2uzu+Xm0pZtYm3/Q0SZyjfi2M1xCymjP393yZreNQuNaAkmu7xem2zjSs2Coziq5PB975hnVtkhainuU6ZYbjfrFGS9scXZel9uqjARfft/MLyPExgcETAnCwx4QMCWYqBlfn6nhC3/plwDcn8DByfh9U6WzT9LGEZmcg55eBo9oBbhtkmkkIp0vjsIzcgdzDW/qiVl91sc7SpvNn0tq9OPusKyyMc/v6YgBgCMz3kbycSKMXY1n879UJCGLxP6u++8Nzep2zmYsbXNZKACoFb0paYVEWrS6vbJy2n/HlOVC21+Xcl27b7fWfbLRxo43u/t9W4aKyn4V9Zx+6mM+KencmaXx9pqJkmM3L0v1eUrPz08pNm4I6Z5X4OdguaH19D757IXx9mee+6RqW1vzLsQf/9Efj7eTUkXtt3raz+N9LMxIG69w33X2CG/2gIApQXjYAwKmBOFhDwiYEkzUZ8/zHP2RtruNoGtu74y30+yICDqKCrPiD+yX9ox/WSmRz5747y0t6TR8Fl9MTBRehfy1HkV+RZGOTmMma8/QJ0zP3FzTZXdXiYqrlr3PNzD+MGfcpYb2Y9pMuEySsyWTrFjnIf2TP5wbmq9I82FFGtmtXqfIr65ZH+jQ+K9duqzabqx7kYcmRxcaVck6Tfhf+PjHVNuHLjwx3r5712ee2SjNDtUgKBo69jRlpfV2d/SxKRvv3JNP+W0SqwCA5z753Hh7s61zRP7wz3463n7ruh9jvayp5bkVn6nnDD147/mx5aYZ4c0eEDAlCA97QMCUYKJm/HA4xI3btw4eCJmOiyayrEsm4s6up2BSE4aX9bzpHhU1bZFxtFfi2/ba2sy+veVNa6ONgdUGjYtEEiJD3zUoKWRgKBI2H7NMuyFbpDk2X/Vm/ExVR0ut73hKyo5RuNInJdpY806Z5IYCzNltoO9VjEDFIiWqtE36crHmKTbWO7+xphNEcoqC3CBXDtBRc0XSpa+Y4refff7Z8fYZIxayfvum73/THzs2kWYxuY5zRhhiadYnxpx66lnV9omPempvueHPeW1du2j/6Yd/Mt5+9Z2rqu2tm54GzMndyrva1diiyrA2yq+5tR95Z/UKGeHNHhAwJQgPe0DAlCA87AEBU4KJ+uxZnqPVPliaanHeh0re2dxWbXe3fPYZh8EWC6a0bp/oMKOFzj5rgcJPbQlhUMhtyVBvnEnHwgqpyQZr7vlzLJn+hfqwdcm6dG6zVBp4fmVJ7Tck2m9vb0+1sS+qdDuNj+f4d96KVhL9WCc68+zqab0fZcTt7unw5F1yN3eo3HIn1eNoEZWlhCwA1Ct+baVB53XhSS088elf8H7z6z/9sWrr0PpDma77ghHbiKp+vp/7qM5K+9Rznxhvxya7b0iTfIlEJf/oR6+o/V69fN1/R8w7NvHrM0ICHntDPR8d0opfqOh1nLnRuk5sF3EI4c0eEDAlCA97QMCUYKJmfBzHmJudPbCtMuPNkuuXdUnbNoVjOYpW65nsJ+eolLEJ0ROO9iJzqGh+7hxRamXTuEOm6ixZvi1DO3FGUtVEQSUDP66eMVu5NLOjMLFOR/dfIyquMaezyLZ3PD3TibyJXytpl4f1962WX53KEC9ThGGjobPSrpI+/k5Hn8sd0jjfoeg3Z1VLyM2xbkKBhC3OURTbl0aZk/cwV/dz/OYrOnIyomsxO+NN9/lZbQZ/6Lx3BZ48vaza3nzzzfH25Zs3VdsGlXPeavrr1Mn0eXaJUusOtNs0M+PvTaHvZZmOctyhTMJPPveLqq0i+/d0MYhXBAQEhIc9IGBKMFkNujjGwsiMt4kwfUoQKZe1ybnb9aZSpGSOTRVU0qCzcsMFjv4ijbiiiZZi8YeBETvrtb2JnJBbMDSZGUNKqui2tdZZjcz6p5f1KnuJzrtJkX1bW7rc0eysN0fPn9JljMo0Py1aqXdWnptM5NMLOhmoQqvgW+S63GxeV/vt7Pm5WtvTrsb2nh9/RuIeRSMuUSeXpGauRY3muFH3czMc6GNVY9/2i89+WLVdv+aj02rkCsyZSLs337003n719Z/pPm75Vfa2VV0h1mSPKtkWiyaJZcbr65VNWSde0R9QJF9kIha5bNS8Ecf4+PnnAehqtBbhzR4QMCUID3tAwJQgPOwBAVOCifrscG5cPscEdGFA0W8VQxPNVLyPMyRqTJLDSxolRgtdKF6tXGZ9ee2Xs2CfM744a9azoOLSgqYTu86Pd9ZEOj3zlBdTqNd0FNf1Gz4jsEyUVGaEODZJ47xWNaKEK140YWm+4b9jMsp4Dio1Td+12t7Xv7XpBSS2dvfMfiQSapKtkjKNi8som/keDmh9o6XPk0s2N6lM849/8oraT37BC1YszmvarENjXDzl29p9ne3Irnixoq/L3KKnBEvmvt2gcZVoHitlfV0SEjwtmOjOTerDUXTd0EQ2NmnMr7z2umr79Mf3BS2HR4hXHLc++xUALQAZgNQ594KILAD4vwE8BeAKgP/GObd9WB8BAQEni/djxv+yc+5559wLo8/fAPCyc+4CgJdHnwMCAj6geBgz/isAPj/a/jb2a8B9/agvRAJURuZpZn5mmEyJTGJJjSqf9lj8wWSSsAl0X3kpNv9Jd11yzaVUSmRimaqiNXIbTlEpoUZdm+rMLq0u66izKkVxNfs66iwp+/E/fcrTcjvGfN7Y9olB3Z5OQOlRotFMzZuS8ybxY2PH93Gpo03aAVFDuy26MmbCF+Y8nbRDOnMAMODyW7RdiPQtVyc3pFzS5i2b+J0hRUd29cX94x97k3Z1VdOZzM6eKvhjDVsmKrHiTfCOaVsgrXtL2V296V2vdfpeakzwvS7RoJGmHwc0rxk5FG2j65eSfv3VdU3H5rKv39cx32Ec983uAPxHEflTEXlx9LdTzrnbADD6f+WYfQUEBJwAjvtm/5xz7paIrAD4voi8+cBvjDD6cXgRABYb8w/YOyAg4HHhWG9259yt0f9rAH4XwGcA3BWRVQAY/b92yHdfcs694Jx7oV6rHbRLQEDABPDAN7uI1ABEzrnWaPuvAviHAL4H4KsAvjX6/7sP7AviKTHjU5897SmjjR3tj9yhMraq1LChcdjfsbWwYvLnG0R51UxWWkT13bpt7Ssvkg+8VPfbpYI+mfkF76dXTfhis+X7vLGuxReLFHq5Sj770oIuZdxXNKUe/5Bq5u1S1tTZs1rwgX32jvH7OWSYxRc/+fHn1H4sgPGzd99Tbbe2/TUcUjZizfzg83zbenE1Ft2kLLWSydJjzfqbdzUh5Og+uH7Hi0DutvU5P/2Ep0R3NjZV2/kn/NzNmfuFM/M6FJ7cHeq1oIhEJeJE3y+s/c8lxO16EigcnOcXAMr1/XWFof0O4Thm/CkAvztSs0wA/F/Ouf8gIj8C8B0R+RqAawB+9Rh9BQQEnBAe+LA7594D8NwBf98E8MXHMaiAgIBHj4lG0EkEFEZlmCKr+U6RPwMT3TQ/QxpdRPHkiaYw2KzsmrK7ykQkKigb6GNlZFI1KtpkO7PYGG+XiCaqkR4dAHToXF5/7W3VdueuNyWLJT3951m8gczx2ERFnV70EXtZpOmqnaafH9YWt+bzOXKbLl3ROubLlB32Fy54Pbalho60yyhbqx49qdre2/Rm5u0db942jWYel9uyOvq1GlGpdM26lpIiqtb2wS5Pt+u3+wNNq26/4decnzKCICt1/7lc1FGbCfmjBc7INPu1SIfPmTLeVYoY5WuW1PU42BVomWfk1sgtOyqCLsTGBwRMCcLDHhAwJQgPe0DAlGCiPnua5dgwoZ/30CZFlKRYPnAfAFghGqpqssbWNjxFVzAViYX870aFFG36moJZIn/YCiA25nwbl2Le2NOhom+98+54u9O3JZW9v9Yw2VWs0OPIFxSTJVWgcs63buvwhj1SjMnIl213tGJOm7L2Vmp6vn+R/PQzc0QxQk+qowzB8ildn2+T6ta1S36/Yqmh9ttuEl2Vm7LPtG7RI7qtaLTb2Uu12vM9osAiuq9KBe3bD1o74+2qCds9RdlyXVMmXIgOK7LvDT3GdOjPs2j8ec6Q22s1D91vSGHMkdGH7/X32/IjqLfwZg8ImBKEhz0gYEow2ZLNaY47W+0D2ypFP5TE0HKVsm8rU8RYakQUa2R+rTS0icyiFBUStqiWtfm5QqWWcpOddGPbR2ddI/N5fXNH7cdRbHUT4fbEE+f9sVZ17hDTRLdIZHJgss1+dtVrl99Z06WBWYbhNGm+zxnqrUrz8dyHNW325Ir/XlG8kSwmGxEkiBgbs5X3PXvau0OZyXrrtHXkHaNE7gtnc7X7+v7oE63aNfr1HEHHpbStvnpp3ruHS0s6YtFRSbDXLunxvnvHC1pmzp9bL9XUGLtvtnw2021MMbaNWGmdIgqtOEtzY/8+yKywKCG82QMCpgThYQ8ImBJMuIqrw2533+SKzKpmseDNXSs8kVBkUqfjVzWj+4QQ/OrloGdKMvUp2mvRJ6rMLzTUfs2mXw3d2tVm1JU1b7pv7np3pGY03BYa3gx8msx2ACjT6u0dYg8AoEVlnppUVmjbVEjdJE12ZxZf2VzniK5EtOl4ZtWvMD9t3IlZWu3mFWGXaRMxpZJGidEDXCFN/Kvr3v25azTw2XWJTdVcNn13ifGwJaSGZLpGRtxwhjQAHbkWYt5zdoWf0SRBkPeoUisArNM90pj17o+tpspWty23FZOYymyFqvD29cWdI1WUWZNgtTDSNvyD3z+cyQpv9oCAKUF42AMCpgThYQ8ImBJM1Gd3AmT39NBN5k9MVMgg15FaHaKyilT/q1TUw+fMtr4Rc1xZ8X5pXPb+zhUTgXbthq9nlpnMuUbdR9A9+REfZTY3r+W2SjPeh1/baqq2S9e9QOHlG3dU264qbUwijSXtD3MNMJdqX7w/8P597Lz/xlGDALAy46O2EuhrUaA55jWGvKDnW1igoaD7P0UU5jt3vEhH09SEi+lcrODIgK4nU3vOrvdQ5FqlbKLTiPrM6L6yx+r0/LVeb+oozzm6Li2TLQeiB5lSi406S5V88VKkx1+npYrlZb+WEqd6jB955qnx9vPPfky17Y1out+cCbXeAgKmHuFhDwiYEky2/BOAfGSCOdGmDJfRmTOmGDKinoieKVn9NSrlO2uUbFnb684dbz7vtTUVdGbVizqcP3dOtZ0mrXhQeaa1Ta17doXM87eu3lBtNzb8vrtt7a4w9cS0mY3kyyiirmjmsUTm6TLptp1Z1FFhc1WeO21y7pJJm5N5PjA6/Ztk7t7d0ZTUu2tex22D5qdiShl3yCw2bBWKdM0SEoC3JrgqL2USQXo9HcnmD2bKgxFll5t64rskPNHta9eOv8fjb5gkrSG5hEVDD14g8/wMuZt5zxyLNAUvva1FUTZHtHC3o90kRnizBwRMCcLDHhAwJQgPe0DAlGCiPnueO09xiPa7ekTFJcZHLZT855hEJm320IBoqMhSUls+NLUU+WP9yi//V2q/U0R99Ieavtsk4Y23L/vMs1cvXVf7XSN/tWlCXUEhvlYEMqZ1gIjP01CABaKrioamfO7DXv/8OaIHZ4xWeUYiD5tt3f8Wadunazvj7Z2WpqQ2d3yo6N1tTTEOKKS1VPI0X80IcZRpHAMT+sv+8IB8ew6fBoAhhfEavUklLMk17JJEv+c4s41LO+/34a+hGMpY6NgprQ80TukQ5MVZvxZUN0KSPOZXXrs03m41dbh2RGsVC6a+4OxoDSYL4hUBAQHhYQ8ImBJMWDde7tNKv4c+0RudgTYrZ0mrLSWaKO1qMzuizKvdPW0CnTuzOt5+/mMf8WOKtVn2NpUxWt/eUW3rVJbq1Xe96b6+p8ebRp5eKlQ1BcPaYYnhmoQiq5w7XP+b26pVk/007zP62NzdbGvRkM1N72p0u3r865SZttPnssl6TDndPpGJoCsQjSjklomhnapk4sfGXSlSqSWpkWtnssY6VHK6aVwNPh67SbDsHQ0rs6XDyOQ/vaTFTiC+zwpRhbasGGdT3rqpIyfvbHlq8p6W3D70eVYoUjA16Y6Z7B/bBN0pHOvNLiINEfk3IvKmiLwhIn9RRBZE5Psicmn0fyjRGhDwAcZxzfj/FcB/cM59DPuloN4A8A0ALzvnLgB4efQ5ICDgA4rjVHGdBfCXAfx3AOCcGwAYiMhXAHx+tNu3AfwAwNeP6isWGWuyWYGKIWmM9QZ2FdybLOWqX+W0eltp6r9XMGIKCyvejL/4Jq14tnXEUX/gzag767qa55Ai1wbi+4+Kmj0o07EzY25xgktsot8iqjTLMtCRKfHEK8JDo0/3k0texprdjsQIQ1y96dmEzITGZXRtOHrPGfM5px2jzCaxeLesTedZtKvxxDqUjS4cuyGDnr/We+aaDdPDXR4usKuSZKwLlXkXYtuIljz7Ue/2JSXNoPSHflxbxPhcvnZL7ZeSa9A2DA2XrIoLJEdtEmaYoemZhJzuKGkot2om/P1DWzyeAbAO4P8UkR+LyP8xKt18yjl3GwBG/68c1UlAQMDJ4jgPewLg0wD+d+fcpwC08T5MdhF5UUQuisjFTvfwuN2AgIDHi+M87DcA3HDO/XD0+d9g/+G/KyKrADD6f+2gLzvnXnLOveCce6FaOTzXNiAg4PHiOPXZ74jIdRH5qHPuLezXZP/Z6N9XAXxr9P93H9gXHLJxxJF22htUJnfPlPVtU6bRjbv+N8XqyycF/9tlNeX/n//8//n+aH2gOqOzwXbJHxwYCpDpJBZirJS1z87fk1T7UJzNVjR+Y8aCjuQCJ8Z3S5Q4om5r9fyaww5RUmmus7829/wczBjBTD63jAbSMRlVfJ62HBHPVYFoucj4/UNaf4hTU7Or569vh2jW3PrbRK8NjbZ9hcYxQ7rr9br2vW/f8T42l70GgLff8yWtt7d1hiOLXiSUhZkUtPBjj+45MW1cF4GjKlNzLr2uH1e7b6JHN/bXl+x9zzguz/4/AvhtESkCeA/Af499q+A7IvI1ANcA/Oox+woICDgBHOthd869AuCFA5q++EhHExAQ8NgwYfEKgTMRVPdQ5Igruw9LnSXeBLKmY0aRZbtGtGCvS5FglEBTGeg+mO4pUnQXAMREjdWoIujy0qzaLx96025oos42iZ7pWTeBElzqVd8/a+sBQKnA++kxJtRHj2ihXPR5qoqxRsihN2TznOg1aNQqvo/YaPgPyCVhutG6RnwuRROFx2bsUX0wj2uTZFKlKe//bueUr9Omiczcbvl7x9YqYAqP3bD+ULsCOdFrXA0Y0HRki6rfDk2iV04UY72iI/SKo+spcvgyXIiNDwiYEoSHPSBgShAe9oCAKcFkdeOdU1rgh8H64gn5NJxd1TWZbaw9Pzu/pNpQ8iGKTarZ1jFUSrVCQgtlTZFwvS7Jvd9YLWj6a7bG/vyCbiMf+9ZdU26ZQlPrJFjYMqKYZdJ1n5/TtFmbShb3aHvOCHC2KAvO+sAszlkgH7BoxDY42pdFRQCgtennO0lojcQITuq1Gt1Hl6gmZuVyk7KWkn59P9X+9lKjMd5uzPttDs+2KBV1PAiHqXYt5UVZahnVbMsMJcraIRySDQA9yvhkgcxiSY8jobDsxMzjvWvoHjJcNiAg4M8BwsMeEDAlkKNe+4/8YCLrAK4CWAKw8YDdJ4EwDo0wDo0Pwjje7xiedM4tH9Qw0Yd9fFCRi865g4J0wjjCOMI4HtMYghkfEDAlCA97QMCU4KQe9pdO6LgWYRwaYRwaH4RxPLIxnIjPHhAQMHkEMz4gYEow0YddRL4sIm+JyDsiMjE1WhH5LRFZE5HX6G8Tl8IWkfMi8ocjOe7XReQ3TmIsIlIWkT8RkZ+MxvEPTmIcNJ54pG/4eyc1DhG5IiKvisgrInLxBMfx2GTbJ/awy34s5P8G4K8BeBbAr4nIsxM6/L8A8GXzt5OQwk4B/B3n3McBfBbAr4/mYNJj6QP4gnPuOQDPA/iyiHz2BMZxD7+BfXnyezipcfyyc+55orpOYhyPT7bdOTeRfwD+IoA/oM/fBPDNCR7/KQCv0ee3AKyOtlcBvDWpsdAYvgvgSyc5FgBVAH8G4JdOYhwAzo1u4C8A+L2TujYArgBYMn+b6DgAzAK4jNFa2qMexyTN+LMAuNzpjdHfTgonKoUtIk8B+BSAH57EWEam8yvYFwr9vtsXFD2JOflnAP4udEGmkxiHA/AfReRPReTFExrHY5Vtn+TDfpBEzVRSASIyA+B3APxt59zug/Z/HHDOZc6557H/Zv2MiHxi0mMQkb8OYM0596eTPvYB+Jxz7tPYdzN/XUT+8gmM4aFk2x+EST7sNwCcp8/nANw6ZN9J4FhS2I8aIlLA/oP+2865f3uSYwEA59wO9qv5fPkExvE5AH9DRK4A+NcAviAi//IExgHn3K3R/2sAfhfAZ05gHA8l2/4gTPJh/xGACyLy9Eil9m8C+N4Ej2/xPexLYAPHlMJ+WMi+uN4/B/CGc+43T2osIrIsIo3RdgXAXwHw5qTH4Zz7pnPunHPuKezfD/+vc+5vTXocIlITkfq9bQB/FcBrkx6Hc+4OgOsi8tHRn+7Jtj+acTzuhQ+z0PArAN4G8C6Avz/B4/4rALcBDLH/6/k1AIvYXxi6NPp/YQLj+EvYd11+CuCV0b9fmfRYAHwSwI9H43gNwP8y+vvE54TG9Hn4BbpJz8czAH4y+vf6vXvzhO6R5wFcHF2bfwdg/lGNI0TQBQRMCUIEXUDAlCA87AEBU4LwsAcETAnCwx4QMCUID3tAwJQgPOwBAVOC8LAHBEwJwsMeEDAl+P8BtBYUNVhwhGUAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "#Print the corresponding label for the image\n",
    "\n",
    "random_image = train_dataset[13421][0].numpy().transpose((1, 2, 0))   \n",
    "mean = np.array([0.485, 0.456, 0.406])\n",
    "std = np.array([0.229, 0.224, 0.225])\n",
    "random_image = std * random_image + mean\n",
    "random_image = np.clip(random_image, 0, 1)\n",
    "print(\"Image label:\", train_dataset[13421][1])\n",
    "plt.imshow(random_image)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6a6d205c",
   "metadata": {},
   "source": [
    "# Convolutional Neural Network Architecture "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "f167ce4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create the model class\n",
    "class CNN(nn.Module):\n",
    "    def __init__(self):\n",
    "        super(CNN,self).__init__()\n",
    "        #Same Padding = [(filter size - 1) / 2] (Same Padding--> input size = output size)\n",
    "        self.cnn1 = nn.Conv2d(in_channels=3, out_channels=4, kernel_size=3,stride=1, padding=1)\n",
    "        #The output size of each of the 4 feature maps is \n",
    "        #[(input_size - filter_size + 2(padding) / stride) +1] --> [(64-3+2(1)/1)+1] = 64 (padding type is same)\n",
    "        self.batchnorm1 = nn.BatchNorm2d(4)\n",
    "        self.relu = nn.ReLU(inplace=True)\n",
    "        self.maxpool1 = nn.MaxPool2d(kernel_size=2)\n",
    "  \n",
    "        #After max pooling, the output of each feature map is now 64/2 =32\n",
    "        self.cnn2 = nn.Conv2d(in_channels=4, out_channels=8, kernel_size=3, stride=1, padding=1)\n",
    "        #Output size of each of the 32 feature maps\n",
    "        self.batchnorm2 = nn.BatchNorm2d(8)\n",
    "        self.maxpool2 = nn.MaxPool2d(kernel_size=2)\n",
    "        \n",
    "        #After max pooling, the output of each feature map is 32/2 = 16\n",
    "        #Flatten the feature maps. You have 8 feature maps, each of them is of size 16x16 --> 8*16*16 = 2048\n",
    "        self.fc1 = nn.Linear(in_features=8*16*16, out_features=32)\n",
    "        self.droput = nn.Dropout(p=0.5)\n",
    "        self.fc2 = nn.Linear(in_features=32, out_features=10)\n",
    "        \n",
    "    def forward(self,x):\n",
    "        out = self.cnn1(x)\n",
    "        out = self.batchnorm1(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.maxpool1(out)\n",
    "        out = self.cnn2(out)\n",
    "        out = self.batchnorm2(out)\n",
    "        out = self.relu(out)\n",
    "        out = self.maxpool2(out)\n",
    "        \n",
    "        #Now we have to flatten the output. This is where we apply the feed forward neural network as learned before! \n",
    "        #It will take the shape (batch_size, 2048)\n",
    "        out = out.view(x.size(0), -1)\n",
    "        \n",
    "        #Then we forward through our fully connected layer \n",
    "        out = self.fc1(out)\n",
    "        out = self.relu(out)\n",
    "        #out = self.droput(out)\n",
    "        out = self.fc2(out)\n",
    "        return out"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "dd9e37ce",
   "metadata": {},
   "source": [
    "# Create models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "25429dcc",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CNN\n",
      "Network OK\n"
     ]
    }
   ],
   "source": [
    "# Create network\n",
    "model = CNN()\n",
    "if USE_CUDA:\n",
    "    model = model.cuda()  \n",
    "    \n",
    "print('Network OK')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "711b61a5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define criterion, optimizer, and scheduler\n",
    "\n",
    "criterion = nn.CrossEntropyLoss()  \n",
    "optimizer = torch.optim.SGD(model.parameters(), lr=LEARNING_RATE, momentum=MOMENTUM, weight_decay=WEIGHT_DECAY)\n",
    "scheduler = torch.optim.lr_scheduler.StepLR(optimizer, step_size=7, gamma=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c0ca5c04",
   "metadata": {},
   "source": [
    "# Training with CNN"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3299f44",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Main loop\n",
    "train_loss = []\n",
    "train_accuracy = []\n",
    "test_loss = []\n",
    "test_accuracy = []\n",
    "epochs = []\n",
    "\n",
    "for epoch in range(1, NUM_EPOCHS+1):\n",
    "    print(f'\\n\\nRunning epoch {epoch} of {NUM_EPOCHS}...\\n')\n",
    "    epochs.append(epoch)\n",
    "\n",
    "    #-------------------------Train-------------------------\n",
    "    \n",
    "    #Reset these below variables to 0 at the begining of every epoch\n",
    "    correct = 0\n",
    "    iterations = 0\n",
    "    iter_loss = 0.0\n",
    "    \n",
    "    model.train()  # Put the network into training mode\n",
    "    \n",
    "    for i, (inputs, labels) in enumerate(train_loader):\n",
    "       \n",
    "        if USE_CUDA:\n",
    "            inputs = inputs.cuda()\n",
    "            labels = labels.cuda()        \n",
    "            \n",
    "        outputs = model(inputs)\n",
    "        loss = criterion(outputs, labels)\n",
    "        iter_loss += loss.item()  # Accumulate the loss\n",
    "        optimizer.zero_grad() # Clear off the gradient in (w = w - gradient)\n",
    "        loss.backward()   # Backpropagation \n",
    "        optimizer.step()  # Update the weights\n",
    "        \n",
    "        # Record the correct predictions for training data \n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        correct += (predicted == labels).sum()\n",
    "        iterations += 1\n",
    "        \n",
    "    scheduler.step()\n",
    "        \n",
    "    # Record the training loss\n",
    "    train_loss.append(iter_loss/iterations)\n",
    "    # Record the training accuracy\n",
    "    train_accuracy.append((100 * correct / len(train_dataset)))   \n",
    "     \n",
    "    #-------------------------Test--------------------------\n",
    "    \n",
    "    correct = 0\n",
    "    iterations = 0\n",
    "    testing_loss = 0.0\n",
    "    \n",
    "    model.eval()  # Put the network into evaluation mode\n",
    "    \n",
    "    for i, (inputs, labels) in enumerate(test_loader):\n",
    "\n",
    "        if USE_CUDA:\n",
    "            inputs = inputs.cuda()\n",
    "            labels = labels.cuda()\n",
    "        \n",
    "        outputs = model(inputs)     \n",
    "        loss = criterion(outputs, labels) # Calculate the loss\n",
    "        testing_loss += loss.item()\n",
    "        # Record the correct predictions for training data\n",
    "        _, predicted = torch.max(outputs, 1)\n",
    "        correct += (predicted == labels).sum()\n",
    "        \n",
    "        iterations += 1\n",
    "\n",
    "    # Record the Testing loss\n",
    "    test_loss.append(testing_loss/iterations)\n",
    "    # Record the Testing accuracy\n",
    "    test_accuracy.append((100 * correct / len(test_dataset)))\n",
    "   \n",
    "    print(f'\\nEpoch {epoch} validation results: Loss={test_loss[-1]} | Accuracy={test_accuracy[-1]}\\n')\n",
    "\n",
    "    # Plot and save\n",
    "    plt.figure(figsize=(12, 8), num=1)\n",
    "    plt.clf()\n",
    "    plt.plot(epochs, train_loss, label='Train')\n",
    "    plt.plot(epochs, test_loss, label='Test')\n",
    "    plt.legend()\n",
    "    plt.grid()\n",
    "    plt.title('Cross entropy loss')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Loss')\n",
    "    plt.savefig('outputs/01-loss-cnn.pdf')\n",
    "\n",
    "    plt.figure(figsize=(12, 8), num=2)\n",
    "    plt.clf()\n",
    "    plt.plot(epochs, train_accuracy, label='Train')\n",
    "    plt.plot(epochs, test_accuracy, label='Test')\n",
    "    plt.legend()\n",
    "    plt.grid()\n",
    "    plt.title('Accuracy')\n",
    "    plt.xlabel('Epoch')\n",
    "    plt.ylabel('Accuracy')\n",
    "    plt.savefig('outputs/02-accuracy-cnn.pdf')"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "252edc3a",
   "metadata": {},
   "source": [
    "# Results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 154,
   "id": "edc6a876",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Final train loss: 0.49078181050938263\n",
      "Final test loss: 0.5623244762858923\n",
      "Final train accuracy: 83.15740966796875\n",
      "Final test accuracy: 80.55555725097656\n"
     ]
    }
   ],
   "source": [
    "print(f'Final train loss: {train_loss[-1]}')\n",
    "print(f'Final test loss: {test_loss[-1]}')\n",
    "print(f'Final train accuracy: {train_accuracy[-1]}')\n",
    "print(f'Final test accuracy: {test_accuracy[-1]}')"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}