resnet50_imagenet.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "resnet50_imagenet.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/akshitadixit/Retinopathy/blob/main/resnet50_imagenet.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Gcfj53WG4ewz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "41f8cd27-e11e-4184-dd17-55780e96ad8a"
      },
      "source": [
        "import os\n",
        "import cv2\n",
        "import csv\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "from sklearn.utils import shuffle\n",
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ErotCjs5RR1C"
      },
      "source": [
        "import tensorflow as tf\n",
        "from tensorflow.keras.layers.experimental import preprocessing\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "from tensorflow.keras import layers\n",
        "from tensorflow.keras import activations\n",
        "from tensorflow.keras.models import Model\n",
        "from tensorflow.keras.layers import Dense,Dropout,MaxPooling2D, GlobalAveragePooling2D\n",
        "from tensorflow.keras.applications import ResNet50\n",
        "import tensorflow.keras as keras"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L4oD1FaakNAa"
      },
      "source": [
        "data_root = \"/content/gdrive/My Drive/Colab Notebooks/dataset/\""
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "trqQiXYI5DkF"
      },
      "source": [
        "IMG_SIZE = 512"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3Atwyi_-5KX7"
      },
      "source": [
        "# Model parameters\n",
        "BATCH_SIZE = 8\n",
        "EPOCHS = 40\n",
        "WARMUP_EPOCHS = 2\n",
        "LEARNING_RATE = 1e-7\n",
        "WARMUP_LEARNING_RATE = 1e-4\n",
        "HEIGHT = 256\n",
        "WIDTH = 256\n",
        "CANAL = 3\n",
        "N_CLASSES = 5\n",
        "ES_PATIENCE = 5\n",
        "RLROP_PATIENCE = 3\n",
        "DECAY_DROP = 0.5"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oa9WwiIg5bqt",
        "outputId": "58198382-b258-4f17-c75c-72f16e2f2dea"
      },
      "source": [
        "import gc\n",
        "gc.collect()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "221"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jQPN0kU0Ut7b"
      },
      "source": [
        "input_tensor = tf.keras.layers.Input(shape=(HEIGHT, WIDTH, CANAL))\n",
        "\n",
        "base = ResNet50(weights=\"imagenet\", include_top=False,input_tensor=input_tensor, input_shape=(HEIGHT, WIDTH, CANAL))\n",
        "#base.load_weights('resnet50_weights_tf_dim_ordering_tf_kernels_notop.h5')\n",
        "\n",
        "top = Dense(N_CLASSES, activation='softmax')(\n",
        "            Dropout(0.5)(\n",
        "                Dense(2048, activation='relu')(\n",
        "                    Dropout(0.5)(\n",
        "                        GlobalAveragePooling2D()(base.output)\n",
        "                    )\n",
        "                )\n",
        "            )\n",
        "        )\n",
        "\n",
        "model = Model(input_tensor, top)"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "XSXHdY21iNwA",
        "outputId": "7f5ba671-3e2c-4184-9346-eb64f261364b"
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"model\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_1 (InputLayer)            [(None, 256, 256, 3) 0                                            \n",
            "__________________________________________________________________________________________________\n",
            "conv1_pad (ZeroPadding2D)       (None, 262, 262, 3)  0           input_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "conv1_conv (Conv2D)             (None, 128, 128, 64) 9472        conv1_pad[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv1_bn (BatchNormalization)   (None, 128, 128, 64) 256         conv1_conv[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv1_relu (Activation)         (None, 128, 128, 64) 0           conv1_bn[0][0]                   \n",
            "__________________________________________________________________________________________________\n",
            "pool1_pad (ZeroPadding2D)       (None, 130, 130, 64) 0           conv1_relu[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "pool1_pool (MaxPooling2D)       (None, 64, 64, 64)   0           pool1_pad[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_1_conv (Conv2D)    (None, 64, 64, 64)   4160        pool1_pool[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_1_bn (BatchNormali (None, 64, 64, 64)   256         conv2_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_1_relu (Activation (None, 64, 64, 64)   0           conv2_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_2_conv (Conv2D)    (None, 64, 64, 64)   36928       conv2_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_2_bn (BatchNormali (None, 64, 64, 64)   256         conv2_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_2_relu (Activation (None, 64, 64, 64)   0           conv2_block1_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_0_conv (Conv2D)    (None, 64, 64, 256)  16640       pool1_pool[0][0]                 \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_3_conv (Conv2D)    (None, 64, 64, 256)  16640       conv2_block1_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_0_bn (BatchNormali (None, 64, 64, 256)  1024        conv2_block1_0_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_3_bn (BatchNormali (None, 64, 64, 256)  1024        conv2_block1_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_add (Add)          (None, 64, 64, 256)  0           conv2_block1_0_bn[0][0]          \n",
            "                                                                 conv2_block1_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block1_out (Activation)   (None, 64, 64, 256)  0           conv2_block1_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_1_conv (Conv2D)    (None, 64, 64, 64)   16448       conv2_block1_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_1_bn (BatchNormali (None, 64, 64, 64)   256         conv2_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_1_relu (Activation (None, 64, 64, 64)   0           conv2_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_2_conv (Conv2D)    (None, 64, 64, 64)   36928       conv2_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_2_bn (BatchNormali (None, 64, 64, 64)   256         conv2_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_2_relu (Activation (None, 64, 64, 64)   0           conv2_block2_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_3_conv (Conv2D)    (None, 64, 64, 256)  16640       conv2_block2_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_3_bn (BatchNormali (None, 64, 64, 256)  1024        conv2_block2_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_add (Add)          (None, 64, 64, 256)  0           conv2_block1_out[0][0]           \n",
            "                                                                 conv2_block2_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block2_out (Activation)   (None, 64, 64, 256)  0           conv2_block2_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_1_conv (Conv2D)    (None, 64, 64, 64)   16448       conv2_block2_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_1_bn (BatchNormali (None, 64, 64, 64)   256         conv2_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_1_relu (Activation (None, 64, 64, 64)   0           conv2_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_2_conv (Conv2D)    (None, 64, 64, 64)   36928       conv2_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_2_bn (BatchNormali (None, 64, 64, 64)   256         conv2_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_2_relu (Activation (None, 64, 64, 64)   0           conv2_block3_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_3_conv (Conv2D)    (None, 64, 64, 256)  16640       conv2_block3_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_3_bn (BatchNormali (None, 64, 64, 256)  1024        conv2_block3_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_add (Add)          (None, 64, 64, 256)  0           conv2_block2_out[0][0]           \n",
            "                                                                 conv2_block3_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv2_block3_out (Activation)   (None, 64, 64, 256)  0           conv2_block3_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_1_conv (Conv2D)    (None, 32, 32, 128)  32896       conv2_block3_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_1_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_1_relu (Activation (None, 32, 32, 128)  0           conv3_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_2_conv (Conv2D)    (None, 32, 32, 128)  147584      conv3_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_2_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_2_relu (Activation (None, 32, 32, 128)  0           conv3_block1_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_0_conv (Conv2D)    (None, 32, 32, 512)  131584      conv2_block3_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_3_conv (Conv2D)    (None, 32, 32, 512)  66048       conv3_block1_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_0_bn (BatchNormali (None, 32, 32, 512)  2048        conv3_block1_0_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_3_bn (BatchNormali (None, 32, 32, 512)  2048        conv3_block1_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_add (Add)          (None, 32, 32, 512)  0           conv3_block1_0_bn[0][0]          \n",
            "                                                                 conv3_block1_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block1_out (Activation)   (None, 32, 32, 512)  0           conv3_block1_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_1_conv (Conv2D)    (None, 32, 32, 128)  65664       conv3_block1_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_1_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_1_relu (Activation (None, 32, 32, 128)  0           conv3_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_2_conv (Conv2D)    (None, 32, 32, 128)  147584      conv3_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_2_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_2_relu (Activation (None, 32, 32, 128)  0           conv3_block2_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_3_conv (Conv2D)    (None, 32, 32, 512)  66048       conv3_block2_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_3_bn (BatchNormali (None, 32, 32, 512)  2048        conv3_block2_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_add (Add)          (None, 32, 32, 512)  0           conv3_block1_out[0][0]           \n",
            "                                                                 conv3_block2_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block2_out (Activation)   (None, 32, 32, 512)  0           conv3_block2_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_1_conv (Conv2D)    (None, 32, 32, 128)  65664       conv3_block2_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_1_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_1_relu (Activation (None, 32, 32, 128)  0           conv3_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_2_conv (Conv2D)    (None, 32, 32, 128)  147584      conv3_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_2_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_2_relu (Activation (None, 32, 32, 128)  0           conv3_block3_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_3_conv (Conv2D)    (None, 32, 32, 512)  66048       conv3_block3_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_3_bn (BatchNormali (None, 32, 32, 512)  2048        conv3_block3_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_add (Add)          (None, 32, 32, 512)  0           conv3_block2_out[0][0]           \n",
            "                                                                 conv3_block3_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block3_out (Activation)   (None, 32, 32, 512)  0           conv3_block3_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_1_conv (Conv2D)    (None, 32, 32, 128)  65664       conv3_block3_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_1_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block4_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_1_relu (Activation (None, 32, 32, 128)  0           conv3_block4_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_2_conv (Conv2D)    (None, 32, 32, 128)  147584      conv3_block4_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_2_bn (BatchNormali (None, 32, 32, 128)  512         conv3_block4_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_2_relu (Activation (None, 32, 32, 128)  0           conv3_block4_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_3_conv (Conv2D)    (None, 32, 32, 512)  66048       conv3_block4_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_3_bn (BatchNormali (None, 32, 32, 512)  2048        conv3_block4_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_add (Add)          (None, 32, 32, 512)  0           conv3_block3_out[0][0]           \n",
            "                                                                 conv3_block4_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv3_block4_out (Activation)   (None, 32, 32, 512)  0           conv3_block4_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_1_conv (Conv2D)    (None, 16, 16, 256)  131328      conv3_block4_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_1_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_1_relu (Activation (None, 16, 16, 256)  0           conv4_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_2_conv (Conv2D)    (None, 16, 16, 256)  590080      conv4_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_2_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_2_relu (Activation (None, 16, 16, 256)  0           conv4_block1_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_0_conv (Conv2D)    (None, 16, 16, 1024) 525312      conv3_block4_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_3_conv (Conv2D)    (None, 16, 16, 1024) 263168      conv4_block1_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_0_bn (BatchNormali (None, 16, 16, 1024) 4096        conv4_block1_0_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_3_bn (BatchNormali (None, 16, 16, 1024) 4096        conv4_block1_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_add (Add)          (None, 16, 16, 1024) 0           conv4_block1_0_bn[0][0]          \n",
            "                                                                 conv4_block1_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block1_out (Activation)   (None, 16, 16, 1024) 0           conv4_block1_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_1_conv (Conv2D)    (None, 16, 16, 256)  262400      conv4_block1_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_1_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_1_relu (Activation (None, 16, 16, 256)  0           conv4_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_2_conv (Conv2D)    (None, 16, 16, 256)  590080      conv4_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_2_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_2_relu (Activation (None, 16, 16, 256)  0           conv4_block2_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_3_conv (Conv2D)    (None, 16, 16, 1024) 263168      conv4_block2_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_3_bn (BatchNormali (None, 16, 16, 1024) 4096        conv4_block2_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_add (Add)          (None, 16, 16, 1024) 0           conv4_block1_out[0][0]           \n",
            "                                                                 conv4_block2_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block2_out (Activation)   (None, 16, 16, 1024) 0           conv4_block2_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_1_conv (Conv2D)    (None, 16, 16, 256)  262400      conv4_block2_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_1_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_1_relu (Activation (None, 16, 16, 256)  0           conv4_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_2_conv (Conv2D)    (None, 16, 16, 256)  590080      conv4_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_2_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_2_relu (Activation (None, 16, 16, 256)  0           conv4_block3_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_3_conv (Conv2D)    (None, 16, 16, 1024) 263168      conv4_block3_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_3_bn (BatchNormali (None, 16, 16, 1024) 4096        conv4_block3_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_add (Add)          (None, 16, 16, 1024) 0           conv4_block2_out[0][0]           \n",
            "                                                                 conv4_block3_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block3_out (Activation)   (None, 16, 16, 1024) 0           conv4_block3_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_1_conv (Conv2D)    (None, 16, 16, 256)  262400      conv4_block3_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_1_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block4_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_1_relu (Activation (None, 16, 16, 256)  0           conv4_block4_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_2_conv (Conv2D)    (None, 16, 16, 256)  590080      conv4_block4_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_2_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block4_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_2_relu (Activation (None, 16, 16, 256)  0           conv4_block4_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_3_conv (Conv2D)    (None, 16, 16, 1024) 263168      conv4_block4_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_3_bn (BatchNormali (None, 16, 16, 1024) 4096        conv4_block4_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_add (Add)          (None, 16, 16, 1024) 0           conv4_block3_out[0][0]           \n",
            "                                                                 conv4_block4_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block4_out (Activation)   (None, 16, 16, 1024) 0           conv4_block4_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_1_conv (Conv2D)    (None, 16, 16, 256)  262400      conv4_block4_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_1_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block5_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_1_relu (Activation (None, 16, 16, 256)  0           conv4_block5_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_2_conv (Conv2D)    (None, 16, 16, 256)  590080      conv4_block5_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_2_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block5_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_2_relu (Activation (None, 16, 16, 256)  0           conv4_block5_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_3_conv (Conv2D)    (None, 16, 16, 1024) 263168      conv4_block5_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_3_bn (BatchNormali (None, 16, 16, 1024) 4096        conv4_block5_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_add (Add)          (None, 16, 16, 1024) 0           conv4_block4_out[0][0]           \n",
            "                                                                 conv4_block5_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block5_out (Activation)   (None, 16, 16, 1024) 0           conv4_block5_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_1_conv (Conv2D)    (None, 16, 16, 256)  262400      conv4_block5_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_1_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block6_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_1_relu (Activation (None, 16, 16, 256)  0           conv4_block6_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_2_conv (Conv2D)    (None, 16, 16, 256)  590080      conv4_block6_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_2_bn (BatchNormali (None, 16, 16, 256)  1024        conv4_block6_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_2_relu (Activation (None, 16, 16, 256)  0           conv4_block6_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_3_conv (Conv2D)    (None, 16, 16, 1024) 263168      conv4_block6_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_3_bn (BatchNormali (None, 16, 16, 1024) 4096        conv4_block6_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_add (Add)          (None, 16, 16, 1024) 0           conv4_block5_out[0][0]           \n",
            "                                                                 conv4_block6_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv4_block6_out (Activation)   (None, 16, 16, 1024) 0           conv4_block6_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_1_conv (Conv2D)    (None, 8, 8, 512)    524800      conv4_block6_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_1_bn (BatchNormali (None, 8, 8, 512)    2048        conv5_block1_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_1_relu (Activation (None, 8, 8, 512)    0           conv5_block1_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_2_conv (Conv2D)    (None, 8, 8, 512)    2359808     conv5_block1_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_2_bn (BatchNormali (None, 8, 8, 512)    2048        conv5_block1_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_2_relu (Activation (None, 8, 8, 512)    0           conv5_block1_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_0_conv (Conv2D)    (None, 8, 8, 2048)   2099200     conv4_block6_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_3_conv (Conv2D)    (None, 8, 8, 2048)   1050624     conv5_block1_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_0_bn (BatchNormali (None, 8, 8, 2048)   8192        conv5_block1_0_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_3_bn (BatchNormali (None, 8, 8, 2048)   8192        conv5_block1_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_add (Add)          (None, 8, 8, 2048)   0           conv5_block1_0_bn[0][0]          \n",
            "                                                                 conv5_block1_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block1_out (Activation)   (None, 8, 8, 2048)   0           conv5_block1_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_1_conv (Conv2D)    (None, 8, 8, 512)    1049088     conv5_block1_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_1_bn (BatchNormali (None, 8, 8, 512)    2048        conv5_block2_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_1_relu (Activation (None, 8, 8, 512)    0           conv5_block2_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_2_conv (Conv2D)    (None, 8, 8, 512)    2359808     conv5_block2_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_2_bn (BatchNormali (None, 8, 8, 512)    2048        conv5_block2_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_2_relu (Activation (None, 8, 8, 512)    0           conv5_block2_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_3_conv (Conv2D)    (None, 8, 8, 2048)   1050624     conv5_block2_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_3_bn (BatchNormali (None, 8, 8, 2048)   8192        conv5_block2_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_add (Add)          (None, 8, 8, 2048)   0           conv5_block1_out[0][0]           \n",
            "                                                                 conv5_block2_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block2_out (Activation)   (None, 8, 8, 2048)   0           conv5_block2_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_1_conv (Conv2D)    (None, 8, 8, 512)    1049088     conv5_block2_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_1_bn (BatchNormali (None, 8, 8, 512)    2048        conv5_block3_1_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_1_relu (Activation (None, 8, 8, 512)    0           conv5_block3_1_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_2_conv (Conv2D)    (None, 8, 8, 512)    2359808     conv5_block3_1_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_2_bn (BatchNormali (None, 8, 8, 512)    2048        conv5_block3_2_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_2_relu (Activation (None, 8, 8, 512)    0           conv5_block3_2_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_3_conv (Conv2D)    (None, 8, 8, 2048)   1050624     conv5_block3_2_relu[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_3_bn (BatchNormali (None, 8, 8, 2048)   8192        conv5_block3_3_conv[0][0]        \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_add (Add)          (None, 8, 8, 2048)   0           conv5_block2_out[0][0]           \n",
            "                                                                 conv5_block3_3_bn[0][0]          \n",
            "__________________________________________________________________________________________________\n",
            "conv5_block3_out (Activation)   (None, 8, 8, 2048)   0           conv5_block3_add[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling2d (Globa (None, 2048)         0           conv5_block3_out[0][0]           \n",
            "__________________________________________________________________________________________________\n",
            "dropout_1 (Dropout)             (None, 2048)         0           global_average_pooling2d[0][0]   \n",
            "__________________________________________________________________________________________________\n",
            "dense_1 (Dense)                 (None, 2048)         4196352     dropout_1[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "dropout (Dropout)               (None, 2048)         0           dense_1[0][0]                    \n",
            "__________________________________________________________________________________________________\n",
            "dense (Dense)                   (None, 5)            10245       dropout[0][0]                    \n",
            "==================================================================================================\n",
            "Total params: 27,794,309\n",
            "Trainable params: 4,206,597\n",
            "Non-trainable params: 23,587,712\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f8wzPcEnW2wD"
      },
      "source": [
        "for layer in model.layers:\n",
        "    layer.trainable = False\n",
        "\n",
        "for i in range(-5, 0):\n",
        "    model.layers[i].trainable = True\n",
        "\n",
        "# training only the top layers\n",
        "# non-trainable parameters changed from 53210 to 23587712"
      ],
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "p_VMhUC9VTmG"
      },
      "source": [
        "train = pd.read_pickle(data_root+'train.pkl')\n",
        "train = shuffle(train)\n",
        "train, test = np.split(train, [int(.8*len(train))])"
      ],
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_aT_PxKZWUtg",
        "outputId": "6c82a14a-14f6-403a-a1dc-d5bdcc9bb8b7"
      },
      "source": [
        "# trying image augmentation since to make the model robust, also since training set be very small\n",
        "train_idg = ImageDataGenerator(rescale=1./255, validation_split=0.2, horizontal_flip=True, vertical_flip=True, rotation_range=0.2, zoom_range=0.2)\n",
        "test_idg = ImageDataGenerator(rescale=1./255)\n",
        "\n",
        "train_generator = train_idg.flow_from_dataframe(dataframe=train,\n",
        "                                                      directory=os.path.join(data_root,'train_images_pro/'),\n",
        "                                                      x_col=\"file_name\",\n",
        "                                                      y_col=\"diagnosis\",\n",
        "                                                      batch_size=BATCH_SIZE,\n",
        "                                                      class_mode=\"categorical\",\n",
        "                                                      target_size=(HEIGHT, WIDTH),\n",
        "                                                      subset = 'training')\n",
        "\n",
        "valid_generator = train_idg.flow_from_dataframe(dataframe=train,\n",
        "                                                      directory=os.path.join(data_root,'train_images_pro/'),\n",
        "                                                      x_col=\"file_name\",\n",
        "                                                      y_col=\"diagnosis\",\n",
        "                                                      batch_size=BATCH_SIZE,\n",
        "                                                      class_mode=\"categorical\",    \n",
        "                                                      target_size=(HEIGHT, WIDTH),\n",
        "                                                      subset = 'validation')\n",
        "\n",
        "test_generator = test_idg.flow_from_dataframe(dataframe=test,\n",
        "                                                      directory=os.path.join(data_root,'train_images_pro/'),\n",
        "                                                      x_col=\"file_name\",\n",
        "                                                      target_size=(HEIGHT, WIDTH),\n",
        "                                                      batch_size=1,\n",
        "                                                      shuffle=False,\n",
        "                                                      class_mode=None)\n"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Found 2344 validated image filenames belonging to 5 classes.\n",
            "Found 585 validated image filenames belonging to 5 classes.\n",
            "Found 733 validated image filenames.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VFpd__i_XIZi",
        "outputId": "06760cf2-4fb7-4f92-b6d1-7a7e1ae22302"
      },
      "source": [
        "gc.collect()"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "732"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sW2EcMBsXrvu",
        "outputId": "c8ebddea-c900-411e-ad16-a34c80edef12"
      },
      "source": [
        "model.compile(keras.optimizers.Adam(lr=WARMUP_LEARNING_RATE),loss = 'categorical_crossentropy',metrics = ['accuracy'])"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py:375: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1nOQbLzPX6SU",
        "outputId": "ed335281-4e8b-4970-f383-9795411762fd"
      },
      "source": [
        "STEP_SIZE_TRAIN = train_generator.n//train_generator.batch_size\n",
        "STEP_SIZE_VALID = valid_generator.n//valid_generator.batch_size\n",
        "\n",
        "history_warmup = model.fit(train_generator,\n",
        "                                     steps_per_epoch=STEP_SIZE_TRAIN,\n",
        "                                     validation_data=valid_generator,validation_steps=STEP_SIZE_VALID,\n",
        "                                     epochs=WARMUP_EPOCHS,\n",
        "                                     verbose=1).history"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n",
            "293/293 [==============================] - 76s 235ms/step - loss: 0.8961 - accuracy: 0.6890 - val_loss: 3.2736 - val_accuracy: 0.4932\n",
            "Epoch 2/2\n",
            "293/293 [==============================] - 68s 232ms/step - loss: 0.7408 - accuracy: 0.7376 - val_loss: 1.2595 - val_accuracy: 0.5051\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "CPxakuhZcvqy",
        "outputId": "461dfe1b-1a73-4e5b-f6c7-74ea5e09b133"
      },
      "source": [
        "from keras.callbacks import EarlyStopping,ReduceLROnPlateau\n",
        "\n",
        "es = EarlyStopping(monitor='val_loss', mode='min', patience=ES_PATIENCE, restore_best_weights=True, verbose=1)\n",
        "rlrop = ReduceLROnPlateau(monitor='val_loss', mode='min', patience=RLROP_PATIENCE, factor=DECAY_DROP, min_lr=1e-6, verbose=1)\n",
        "\n",
        "callback_list = [es, rlrop]\n",
        "optimizer = keras.optimizers.Adam(lr=LEARNING_RATE)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/optimizer_v2/optimizer_v2.py:375: UserWarning: The `lr` argument is deprecated, use `learning_rate` instead.\n",
            "  \"The `lr` argument is deprecated, use `learning_rate` instead.\")\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nTox80I3ru3X",
        "outputId": "8741f114-4845-42c7-ebbe-5e79ddd94b0b"
      },
      "source": [
        "history_finetunning = model.fit_generator(generator=train_generator,\n",
        "                                          steps_per_epoch=STEP_SIZE_TRAIN,\n",
        "                                          validation_data=valid_generator,\n",
        "                                          validation_steps=STEP_SIZE_VALID,\n",
        "                                          epochs=EPOCHS,\n",
        "                                          callbacks=callback_list,\n",
        "                                          verbose=1).history"
      ],
      "execution_count": 26,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py:1940: UserWarning: `Model.fit_generator` is deprecated and will be removed in a future version. Please use `Model.fit`, which supports generators.\n",
            "  warnings.warn('`Model.fit_generator` is deprecated and '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/40\n",
            "293/293 [==============================] - 67s 230ms/step - loss: 0.6494 - accuracy: 0.7607 - val_loss: 0.8712 - val_accuracy: 0.7038\n",
            "Epoch 2/40\n",
            "293/293 [==============================] - 67s 229ms/step - loss: 0.5908 - accuracy: 0.7747 - val_loss: 0.5556 - val_accuracy: 0.7877\n",
            "Epoch 3/40\n",
            "293/293 [==============================] - 67s 229ms/step - loss: 0.5782 - accuracy: 0.7969 - val_loss: 0.8326 - val_accuracy: 0.7397\n",
            "Epoch 4/40\n",
            "293/293 [==============================] - 67s 230ms/step - loss: 0.5315 - accuracy: 0.7995 - val_loss: 0.7772 - val_accuracy: 0.7449\n",
            "Epoch 5/40\n",
            "293/293 [==============================] - 67s 228ms/step - loss: 0.5180 - accuracy: 0.7991 - val_loss: 0.7365 - val_accuracy: 0.7003\n",
            "\n",
            "Epoch 00005: ReduceLROnPlateau reducing learning rate to 4.999999873689376e-05.\n",
            "Epoch 6/40\n",
            "293/293 [==============================] - 67s 229ms/step - loss: 0.4608 - accuracy: 0.8208 - val_loss: 0.5136 - val_accuracy: 0.7979\n",
            "Epoch 7/40\n",
            "293/293 [==============================] - 67s 227ms/step - loss: 0.4282 - accuracy: 0.8400 - val_loss: 0.5309 - val_accuracy: 0.8048\n",
            "Epoch 8/40\n",
            "293/293 [==============================] - 67s 228ms/step - loss: 0.3837 - accuracy: 0.8524 - val_loss: 0.5559 - val_accuracy: 0.7945\n",
            "Epoch 9/40\n",
            "293/293 [==============================] - 67s 227ms/step - loss: 0.3714 - accuracy: 0.8554 - val_loss: 0.5914 - val_accuracy: 0.7979\n",
            "\n",
            "Epoch 00009: ReduceLROnPlateau reducing learning rate to 2.499999936844688e-05.\n",
            "Epoch 10/40\n",
            "293/293 [==============================] - 67s 227ms/step - loss: 0.3395 - accuracy: 0.8767 - val_loss: 0.5993 - val_accuracy: 0.8048\n",
            "Epoch 11/40\n",
            "293/293 [==============================] - 67s 228ms/step - loss: 0.3151 - accuracy: 0.8720 - val_loss: 0.5946 - val_accuracy: 0.7928\n",
            "Restoring model weights from the end of the best epoch.\n",
            "Epoch 00011: early stopping\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 350
        },
        "id": "8Sx86mwdr3iK",
        "outputId": "44708749-43c8-453c-aefe-cd342722f575"
      },
      "source": [
        "# import matplotlib.pyplot as plt\n",
        "plt.figure(figsize=(8,5))\n",
        "\n",
        "plt.plot(history_finetunning['accuracy'])\n",
        "plt.plot(history_finetunning['val_accuracy'])\n",
        "plt.title('Model Accuracy')\n",
        "plt.ylabel('Accuracy')\n",
        "plt.xlabel('Epoch')\n",
        "plt.legend(['Train', 'Validation'], loc='upper left')\n",
        "plt.gca().ticklabel_format(axis='both', style='plain', useOffset=False)\n",
        "plt.show()"
      ],
      "execution_count": 27,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAf4AAAFNCAYAAADhMQ3+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdd3xUZfb48c9JJ4UAAUIJofeOAQQLIKIIKnZBV0Vde1n9WlZd2/rT3XXVdW3rLq4rFgQREFEpCoKioBJ6aCFAIKETWgrp5/fHncAQAgTIZDIz5/165TVz65xJlHOfc5/7PKKqGGOMMSYwBHk7AGOMMcZUH0v8xhhjTACxxG+MMcYEEEv8xhhjTACxxG+MMcYEEEv8xhhjTACxxG9MgBKRFiKiIhJSiX1Hi8hP1RGXMcazLPEb4wNEJF1ECkWkfrn1S13Ju4V3IjsqlmgRyRGRGd6OxRhzfJb4jfEdm4BRZQsi0hWI9F44x7gaKACGiEij6vzgylQtjDEOS/zG+I6PgZvdlm8BPnLfQURiReQjEdktIptF5GkRCXJtCxaRV0Vkj4hsBIZXcOz7IrJdRLaKyIsiEnwK8d0C/BtYAfyu3LnPFZEFIrJfRDJEZLRrfS0Rec0V6wER+cm1bqCIZJY7R7qIXOh6/7yITBKRT0TkIDBaRPqIyELXZ2wXkbdFJMzt+M4i8p2I7BWRnSLylIg0EpE8EYlz26+X6/cXegrf3RifYYnfGN/xC1BbRDq6EvJI4JNy+7wFxAKtgAE4Fwq3urbdAVwK9ASSgGvKHTsWKAbauPa5CPh9ZQITkebAQGCc6+fmcttmuGJrAPQAlrk2vwqcBfQH6gGPA6WV+UxgBDAJqOP6zBLgYaA+0A8YDNzriiEGmA3MBJq4vuMcVd0BzAOuczvvTcAEVS2qZBzG+BRL/Mb4lrJW/xBgDbC1bIPbxcCTqpqtqunAaziJDJzk9k9VzVDVvcBf3Y6NB4YBD6lqrqruAl53na8ybgJWqOpqYALQWUR6urbdAMxW1fGqWqSqWaq6zFWJuA34g6puVdUSVV2gqgWV/MyFqjpVVUtV9ZCqLlbVX1S12PXd/4Nz8QPOBc8OVX1NVfNdv59fXds+xFWhcP0OR+H8no3xS3ZfzBjf8jHwI9CScmV+nJZuKLDZbd1moKnrfRMgo9y2Ms1dx24XkbJ1QeX2P5GbgfcAVHWriPyAU/pfCjQDNlRwTH0g4jjbKuOo2ESkHfAPnGpGJM6/b4tdm48XA8CXwL9FpCXQHjigqr+dZkzG1HjW4jfGh6jqZpxOfsOAKeU27wGKcJJ4mUSOVAW24yRA921lMnA65tVX1Tqun9qq2vlkMYlIf6At8KSI7BCRHUBf4AZXp7sMoHUFh+4B8o+zLRe3jouulniDcvuUn1r0XWAt0FZVawNPAWVXMRk4tz+Ooar5wEScVv9NWGvf+DlL/Mb4ntuBC1Q1132lqpbgJLCXRCTGdW/9/zjSD2Ai8KCIJIhIXeAJt2O3A98Cr4lIbREJEpHWIjKAk7sF+A7ohHP/vgfQBagFXIJz//1CEblOREJEJE5EeqhqKfA/4B8i0sTV+bCfiIQDqUCEiAx3dbJ7Ggg/SRwxwEEgR0Q6APe4bfsaaCwiD4lIuOv309dt+0fAaOByLPEbP2eJ3xgfo6obVDX5OJsfwGktbwR+Aj7FSa7glOJnAcuBJRxbMbgZCANWA/twOs41PlEsIhKB03fgLVXd4fazCSeB3qKqW3AqFI8Ae3E69nV3neJRYCWwyLXtZSBIVQ/gdMz7L07FIhc4qpd/BR7F6U+Q7fqun5VtUNVsnH4RlwE7gPXAILftP+N0KlziqqoY47dEtXy1zBhjAo+IfA98qqr/9XYsxniSJX5jTMATkd44tyuauaoDxvgtK/UbYwKaiHyI84z/Q5b0TSCwFr8xxhgTQKzFb4wxxgQQS/zGGGNMAAmIkfvq16+vLVq08HYYxhhjTLVYvHjxHlUtP+gVECCJv0WLFiQnH++xZ2OMMca/iMhxx6OwUr8xxhgTQCzxG2OMMQHEEr8xxhgTQALiHn9FioqKyMzMJD8/39uh+IWIiAgSEhIIDQ31dijGGGNOIGATf2ZmJjExMbRo0QK3+cfNaVBVsrKyyMzMpGXLlt4OxxhjzAkEbKk/Pz+fuLg4S/pVQESIi4uz6okxxviAgE38gCX9KmS/S2OM8Q0Bnfi9KSsrix49etCjRw8aNWpE06ZNDy8XFhae8Njk5GQefPDBaorUGGOMPwnYe/zeFhcXx7JlywB4/vnniY6O5tFHHz28vbi4mJCQiv88SUlJJCUlVUucxhhj/ItHW/wiMlRE1olImog8UcH2RBGZKyJLRWSFiAxzrb9RRJa5/ZSKSA/Xtnmuc5Zta+jJ71CdRo8ezd13303fvn15/PHH+e233+jXrx89e/akf//+rFu3DoB58+Zx6aWXAs5Fw2233cbAgQNp1aoVb775pje/gjHG1Cj78wr5avk2Fm/ex4FDRd4Op0bwWItfRIKBd4AhQCawSESmqepqt92eBiaq6rsi0gmYDrRQ1XHAONd5ugJTVXWZ23E3qqpfjsGbmZnJggULCA4O5uDBg8yfP5+QkBBmz57NU089xeTJk485Zu3atcydO5fs7Gzat2/PPffcY4/VGWMC3qL0vfxh/FK2HTjS8bhhTDhtGkbTtmE0bRpG06ZhDG0aRlM/Oixg+ip5stTfB0hT1Y0AIjIBGAG4J34FarvexwLbKjjPKGCCB+Pkz1+tYvW2g1V6zk5NavPcZZ1P+bhrr72W4OBgAA4cOMAtt9zC+vXrERGKiiq+Wh0+fDjh4eGEh4fTsGFDdu7cSUJCwhnFb4wxvqqkVPnX3DRen51Ks3qRfHx7HwqLS1m/K4e0XTms35XD5CVbySkoPnxMnchQ2jSIpm18NK0bRNM23rkgaBIb4XcXBJ5M/E2BDLflTKBvuX2eB74VkQeAKODCCs5zPc4Fg7sPRKQEmAy8qKpaJRHXAFFRUYffP/PMMwwaNIgvvviC9PR0Bg4cWOEx4eHhh98HBwdTXFxc4X7GGOPvdh3M56HPlrFgQxaXd2/CS1d2ISbCqYAO7hh/eD9VZcfBfOdCYGcOabtzSNuZw8yUHezLO9LIigoLpvXh6kA0bV0VgsR6kQQH+eYFgbc7940CxqrqayLSD/hYRLqoaimAiPQF8lQ1xe2YG1V1q4jE4CT+m4CPyp9YRO4E7gRITEw8YRCn0zKvDgcOHKBp06YAjB071rvBGGNMDTdv3S4embic3MJi/n51N65NSjhua11EaBxbi8axtTiv7dGz12blFByuDKS5fhakZTFlydbD+4SFBNGqftQxFwQt6kcSHhLs0e95pjyZ+LcCzdyWE1zr3N0ODAVQ1YUiEgHUB3a5to8ExrsfoKpbXa/ZIvIpzi2FYxK/qo4BxgAkJSX5ZEXg8ccf55ZbbuHFF19k+PDh3g7HGGNqpKKSUl79dh3/+WEj7eNjmHDD2bSNjznt88VFhxMXHU7fVnFHrT+YX8QG1wVB2euKzAN8s3I7ZXXn4CCheb1IWrv1I2jbMIbWDaOIDPN2W9shnqqSi0gIkAoMxkn4i4AbVHWV2z4zgM9UdayIdATmAE1VVUUkCOdWwXlu/QRCgDqqukdEQnEuCmar6r9PFEtSUpImJx/dF3DNmjV07Nixqr6uwX6nxpjql7E3jwfGL2VZxn5u6JvIs5d2IiK0elvc+UUlbNh9pDpQVi1I35NLcemRHNu0Tq2jOha2jY+mTYMYYiOrvjO2iCxW1Qqf+/bY5YeqFovI/cAsIBj4n6quEpEXgGRVnQY8ArwnIg/jdPQb7Xa//nwgoyzpu4QDs1xJPxiYDbznqe9gjDGm5pq+cjt/nLwCFN65oRfDuzX2ShwRocF0bhJL5yaxR60vKillc1buUf0I1u/M4ZeNWRQUlx7er0FMOJd2a1xtt509WndQ1ek4j+i5r3vW7f1q4JzjHDsPOLvculzgrCoP1BhjjM/ILyrh/329mnG/bqF7szq8PaonzepFejusY4QGB7keF4xhaJcj60tKla37DpG2O9u5INiVQ8OYiGqLq2bccDDGGGMqIW1XNvd/upS1O7K56/xWPHpxe0KDfWv0+eAgITEuksS4SC7oEH/yA6qYJX5jjDE1nqryeXImz01bRWRYMGNv7c3A9n4zcGu1ssRvjDGmRsspKOZPX6zky2Xb6Ncqjn+O7EF87eorjfsbS/zGGGNqrJWZB3hg/BK27M3jkSHtuHdQG58dOKem8K0bI35k0KBBzJo166h1//znP7nnnnsq3H/gwIGUPZI4bNgw9u/ff8w+zz//PK+++uoJP3fq1KmsXn1k1ORnn32W2bNnn2r4xhjjUarK/37axFXv/kxBcSkT7uzHA4PbWtKvApb4vWTUqFFMmHD0FAQTJkxg1KhRJz12+vTp1KlT57Q+t3zif+GFF7jwwopGSjbGGO/Yl1vIHR8l88LXqxnQriHTHzyPPi3reTssv2GJ30uuueYavvnmGwoLCwFIT09n27ZtjB8/nqSkJDp37sxzzz1X4bEtWrRgz549ALz00ku0a9eOc8899/C0vQDvvfcevXv3pnv37lx99dXk5eWxYMECpk2bxmOPPUaPHj3YsGEDo0ePZtKkSQDMmTOHnj170rVrV2677TYKCgoOf95zzz1Hr1696Nq1K2vXrvXkr8YYE8B+27SXS96Yz4+pe3jusk68d/NZ1I0K83ZYfsUSv5fUq1ePPn36MGPGDMBp7V933XW89NJLJCcns2LFCn744QdWrFhx3HMsXryYCRMmsGzZMqZPn86iRYsOb7vqqqtYtGgRy5cvp2PHjrz//vv079+fyy+/nFdeeYVly5bRunXrw/vn5+czevRoPvvsM1auXElxcTHvvvvu4e3169dnyZIl3HPPPSe9nWCMMaeqpFR5c856Ro5ZSERoEFPu7c+t57T0u5nxagLr3Acw4wnYsbJqz9moK1zytxPuUlbuHzFiBBMmTOD9999n4sSJjBkzhuLiYrZv387q1avp1q1bhcfPnz+fK6+8kshIZ+CKyy+//PC2lJQUnn76afbv309OTg4XX3zxCWNZt24dLVu2pF27dgDccsstvPPOOzz00EOAcyEBcNZZZzFlypTK/Q6MMaYSdh7M56EJy1i4MYsrejThxSu7Eh1u6clT7DfrRSNGjODhhx9myZIl5OXlUa9ePV599VUWLVpE3bp1GT16NPn5+ad17tGjRzN16lS6d+/O2LFjmTdv3hnFWjb1r037a4ypSnPX7eLRicvJKyzhlWu6cc1Zx59Rz1QNS/xw0pa5p0RHRzNo0CBuu+02Ro0axcGDB4mKiiI2NpadO3cyY8YMBg4ceNzjzz//fEaPHs2TTz5JcXExX331FXfddRcA2dnZNG7cmKKiIsaNG3d4et+YmBiys7OPOVf79u1JT08nLS2NNm3a8PHHHzNgwACPfG9jjCksdmbUG/PjRjo0iuHtG3rSpuHpz6hnKs8Sv5eNGjWKK6+8kgkTJtChQwd69uxJhw4daNasGeecU+E0Bof16tWL66+/nu7du9OwYUN69+59eNv/+3//j759+9KgQQP69u17ONmPHDmSO+64gzfffPNwpz6AiIgIPvjgA6699lqKi4vp3bs3d999t2e+tDEmoG3JyuOBCUtZnrGf352dyNPDq39GvUDmsWl5axKblrd62O/UGHMy36zYzhOTV4DA36/uxiVdvTOjnr/zyrS8xhhjTJn8ohL+/NVqxv+2hZ6JdXhzZM2cUS8QWOI3xhjjUet3OjPqrduZzd0DWvPIRe18bkY9f2KJ3xhjjEeoKhOTM3hu2iqiwkL48LY+DGjXwNthBbyATvyqao+NVJFA6CtijKm87Pwi/vRFCtOWb+OcNnG8fl0PGtqMejVCwCb+iIgIsrKyiIuLs+R/hlSVrKwsIiLsf2pjDKzI3M8D45eSue8Qj13cnrsHtLbJdWqQgE38CQkJZGZmsnv3bm+H4hciIiJISEjwdhjGGC9SVd7/aRMvz1xLg+hwPrvzbJJa2OQ6NU3AJv7Q0FBatmzp7TCMMcYv7M0t5LHPlzNn7S4u6hTP36/pRp1Im1ynJgrYxG+MMaZq/Loxiz9MWMbe3EL+fHlnbu7X3G6h1mCW+I0xxlSKqrI/r4gdB/PZ6fpZsz2bjxam0zwuiim39KdL01hvh2lOwhK/McYY8gqL2XEgn50HCw4ndff3Ow7ms+tgAYUlpUcdJwJX9UzgzyM624x6PsL+SsYY48eKSkrZle1K4AdcCT27wHmfnc+OA05Czy44dtbNqLBg4mMjiI+JoHeLejSsHU58TASNYiOIrx1OfO0IGsSEEx5i4+z7Ekv8xhjjg0pLlb15hRW2znceLHASenY+e3IKjzk2NFhoGOMk73bxMZzXtgHxtSNoFOsk9vjYCOJrR1gL3k/ZX9UYY2qobfsP8cvGrApL7ruy8ykqOXrgLBGIiwonvnY4jWIj6N6sjvO+tpPIG7re140MI8ieqw9YHk38IjIUeAMIBv6rqn8rtz0R+BCo49rnCVWdLiItgDXAOteuv6jq3a5jzgLGArWA6cAf1IaNM8b4CVXl1017GftzOt+u3kGp61+3mPAQV0s8nL6t6hFfO4L4GCfBN6wdQSNX2d3GwDcn47HELyLBwDvAECATWCQi01R1tdtuTwMTVfVdEemEk8hbuLZtUNUeFZz6XeAO4FfX/kOBGZ75FsYYUz0OFZYwddlWPlyQztod2dSJDOWO81txVc8EEurWIsrK7qaKePK/pD5AmqpuBBCRCcAIwD3xK1Db9T4W2HaiE4pIY6C2qv7iWv4IuAJL/MYYH5WxN49PftnMhEUZHDhURIdGMbx8dVcu796UWmHWac5UPU8m/qZAhttyJtC33D7PA9+KyANAFHCh27aWIrIUOAg8rarzXefMLHfOplUctzHGeJSqsnBDFmMXpDN7zU5EhIs7x3NLvxb0aVnPBr8xHuXt2tEoYKyqviYi/YCPRaQLsB1IVNUs1z39qSLS+VROLCJ3AncCJCYmVnXcxhhzyvIKi/liqVPOT92ZQ93IUO4e0Jrfnd2cJnVqeTs8EyA8mfi3As3clhNc69zdjnOPHlVdKCIRQH1V3QUUuNYvFpENQDvX8e4zwVR0TlzHjQHGACQlJVnnP2OM12zJyuOjhelMTM7gYH4xnZvU5u/XdOPy7k2ICLVyvqlenkz8i4C2ItISJzmPBG4ot88WYDAwVkQ6AhHAbhFpAOxV1RIRaQW0BTaq6l4ROSgiZ+N07rsZeMuD38EYY06LqvJT2h4+XJDOnLW7CBJhaJdGjO7fgqTmda2cb7zGY4lfVYtF5H5gFs6jev9T1VUi8gKQrKrTgEeA90TkYZyOfqNVVUXkfOAFESkCSoG7VXWv69T3cuRxvhlYxz5jTA2SW1DMlCWZfLhwM2m7coiLCuP+QW24sW9zGsVGeDs8Y5BAeAQ+KSlJk5OTvR2GMcaPpe/J5aOFm/k8OYPsgmK6No1ldP8WDO/W2Mr5ptqJyGJVTapom7c79xljjM8qLVXmp+1h7M+bmJe6m2ARhnVtzC39W9ArsY6V802NZInfGGNOUU5BMZMXZ/LhgnQ27smlfnQ4D1zQlhv7JhJf28r5pmazxG+MMZW0cXcOHy3czKTFmeQUFNO9WR1ev747w7o2thnqjM+wxG+MMSdQWqr8kLqbsQvS+SF1N6HBwnBXOb9nYl1vh2fMKbPEb4wxFTiYX8Sk5Ew+WphOelYeDWLCefjCdozq24yGMVbON77LEr8xxrhJ25XNhws2M2VJJrmFJfRKrMPDQ9pxSZfGhIXYzHfG91niN8YEvJJSZd66XYxdkM789XsICw7i0u6NGd2/Bd0S6ng7PGOqlCV+Y0zAOnCoiM+TM/ho4Wa27M0jvnY4jwxpx6i+idSPDvd2eMZ4hCV+Y0zAKS1Vxv22hb9NX0NuYQlJzevy2MXtGdqlEaHBVs43/s0SvzEmoGzJyuPxycv5ZeNezmtbnz8O7UCXprHeDsuYamOJ3xgTEEpLlY8WpvPyzHUEBwl/u6or1/duZqPrmYBjid8Y4/fS9+Ty+OQV/LZpLwPaNeCvV3WlSZ1a3g7LGK+wxG+M8VulpcrYBen8fdZaQoOD+Ps13bj2rARr5ZuAZonfGOOXNu7O4fFJK0jevI8LOjTkL1d2tWlxjcESvzHGz5SUKv/7aROvfruO8JAgXru2O1f1amqtfGNcLPEbY/xG2q4cHpu0nKVb9nNhx3heurKLzZZnTDmW+I0xPq+kVPnv/I289l0qkWHBvDGyB5d3b2KtfGMqYInfGOPT1u/M5tFJK1iesZ+LOsXz4pVdbBIdY07AEr8xxicVl5Tynx838sbs9USFB/PmqJ5c1q2xtfKNOQlL/MYYn7NuRzaPTVrOiswDDOvaiBdGdLGx9Y2pJEv8xhifUVRSyn9+2MAbc9ZTOyKUd27oxfBujb0dljE+xRK/McYnrNl+kEc/X86qbQe5tFtj/nx5Z+KslW/MKbPEb4yp0YpKSvnX3A28PXc9sbVC+ffvejG0i7XyjTldlviNMTXWqm0HePTzFazZfpARPZrw/GWdqRsV5u2wjPFplviNMTVOYXEpb89N419z06gbFcaYm87ios6NvB2WMX7BEr8xpkZJ2XqARz9fztod2VzVsynPXtaJOpHWyjemqgR58uQiMlRE1olImog8UcH2RBGZKyJLRWSFiAxzrR8iIotFZKXr9QK3Y+a5zrnM9dPQk9/BGFM9CopLeHXWOka88zP78gp5/5Yk/nF9D0v6xlQxj7X4RSQYeAcYAmQCi0RkmqqudtvtaWCiqr4rIp2A6UALYA9wmapuE5EuwCygqdtxN6pqsqdiN8ZUrxWZ+3n08+Wk7szhmrMSeGZ4J2IjQ70dljF+yZOl/j5AmqpuBBCRCcAIwD3xK1Db9T4W2Aagqkvd9lkF1BKRcFUt8GC8xphqll9Uwhtz1jPmx400iA7ng1t7M6i9FfGM8SRPJv6mQIbbcibQt9w+zwPfisgDQBRwYQXnuRpYUi7pfyAiJcBk4EVV1SqL2hhTLZZu2cdjk1aQtiuHkb2b8dTwjtSOsFa+MZ7m7c59o4CxqvqaiPQDPhaRLqpaCiAinYGXgYvcjrlRVbeKSAxO4r8J+Kj8iUXkTuBOgMTERA9/DWNMZeUXlfD6d6m8N38jjWpH8OFtfRjQroG3wzImYHgy8W8FmrktJ7jWubsdGAqgqgtFJAKoD+wSkQTgC+BmVd1QdoCqbnW9ZovIpzi3FI5J/Ko6BhgDkJSUZBUBY2qAxZv38dik5WzcncuoPok8NawDMdbKN6ZaeTLxLwLaikhLnIQ/Erih3D5bgMHAWBHpCEQAu0WkDvAN8ISq/ly2s4iEAHVUdY+IhAKXArM9+B2MMVXgUGEJr327jvd/3kST2Fp8cntfzm1b39thGROQPJb4VbVYRO7H6ZEfDPxPVVeJyAtAsqpOAx4B3hORh3E6+o1WVXUd1wZ4VkSedZ3yIiAXmOVK+sE4Sf89T30HY8yZW5S+l8cnrWDTnlxuOrs5f7ykA9Hh3r7LaEzgkkDoF5eUlKTJyfb0nzHVKa+wmFdmrWPsgnQS6tbi5au70b+1tfKNqQ4islhVkyraZpfdxpgqt2b7Qe77dAkbd+cyun8LHru4PVHWyjemRrD/E40xVUZV+WxRBs9NW0VsrVA+vaOvtfKNqWEs8RtjqkRuQTFPT03hi6VbOa9tfV6/vgf1o8O9HZYxphxL/MaYM7ZuRzb3jlvMpj25PDKkHfcNakNQkHg7LGNMBSzxG2NOm6ryeXImz05LISYilHG/P5t+reO8HZYx5gQs8RtjTktuQTHPTE1hytKtnNvGKe03iLHSvjE1nSV+Y8wpcy/t/5+rtB9spX1jfIIlfmNMpakqny/O5NkvU4gOD+WT31uvfWN8jSV+Y0yl5BU6vfanLNnKOW3ieP36HjSMifB2WMaYU2SJ3xhzUqk7s7l33BI27M7h4Qvbcf8FVto3xldZ4jfGnNDnyRk84yrtj7u9L/3bWGnfGF9mid8YU6G8wmKembqKyUsy6dcqjjdGWWnfGH9gid8Yc4z1rtJ+2u4c/jC4LQ8ObmulfWP8hCV+Y8xRJi3O5JmpKUSFB/PJ7X05x0r7xvgVS/zGGAAOFZbwzJcpTFrsKu2P7EHD2lbaN8bfWOI3xrB+Zzb3fbqE9btyeHBwW/5gpX1j/JYlfmMC3OTFmTw9NYXIsGA+uq0P57Vt4O2QjDEeZInfmAB1qLCE56alMDE5k74t6/HmqJ7EW2nfGL9nid+YAJS2y+m1v35XDg9e0IYHB7clJDjI22EZY6qBJX5jAsyUJZn86Qsr7RsTqCzxGxMg3Ev7fVrW4y0r7RsTkCzxGxMA0nblcN+4JaTuyub+QW146EIr7RsTqCzxG+Pnpi7dylNfrCQiNJixt/ZhQLsAL+2XlsLutbBlIexYCQlJ0PEyiIj1dmSmKhUXwLalsHkBZG+HLldDs74g9piqqKq3Y/C4pKQkTU5O9nYYxlSr/KISnp+2igmLMujTsh5vjuxJo9gALO0XFzoJYMtC188vkL/f2RYaCUV5EBwO7YdC1+ug7RAICfduzObU5R+AjN+cRL/lF9i6GEoKnG0hEVCcD/FdIOk26HYdhMd4N14PE5HFqppU4TZL/Mb4n7RdOdz/6RLW7sjmvkGtefjCdoFT2s8/CJm/wWZXkt+a7PyjDxDXBhL7QfP+kHg21G0JmcmwciKkTIG8PU7Lv9MV0PVaaH4OBAXI783XHNwOW1xJfvNC2JkCKASFQOPuzt85sZ/zdw6tBSsnwaL3nCpPWAx0Hwm9b4eGHb39TTzijBK/iFwGfKOqpZ4IrjpY4jeBxL20/4/rujOwfUNvh+RZ2TuclvxmV4t+ZwpoKUgwNO4Gia4kn9gPok9wm6OkCDbOg5Wfw5qvoSgXajd1SsTdrnNai1Ym9g5V2LPeLdEvgP2bnW2hUdCs95G/c0IShEUd/zyZybDov7BqCpQUQvNznQuADpdCSFj1fScPO9PE/wnQD5gM/NULD1cAACAASURBVE9V157CBw8F3gCCgf+q6t/KbU8EPgTquPZ5QlWnu7Y9CdwOlAAPquqsypyzIpb4TSDILyrhz1+tYvxvGfRuUZc3R/WkcWwtb4dVtVQhK+3oRL9vk7MtNNL5R/9wAugN4dGn9zmFubBuBqyYCBvmQGkxNOjgVAG6Xgt1m1fddzLHKimC7SuOJPotCyEvy9kWWR+a9zvSom/UDYJPo7ta7h5Y+gkk/8+5iIiOh163wFmjIbZplX4dbzjjUr+I1AZGAbcCCnwAjFfV7BMcEwykAkOATGARMEpVV7vtMwZYqqrvikgnYLqqtnC9Hw/0AZoAs4F2rsNOeM6KWOI3/m7DbqfX/tod2dw7sDX/N8RPSvslxbBjhSvRu5JA3h5nW2TckVJuYn+ndR8cWvUx5GbB6i9gxeeQ8YuzrtnZ0O1a6HQlRMVV/WcGmoIcyFx0pB9GZrLT9wKc2zFlt2YS+0Nc66qtvJSWQNocSH4fUmeBBEH7S6D376HlAJ+91XOixF+pyyRVPSgik4BawEPAlcBjIvKmqr51nMP6AGmqutEVxARgBOCepBWo7XofC2xzvR8BTFDVAmCTiKS5zkclzmlMQPly2VaemrKSsJAgPri1N4N8ubRfmOv8o1+WADIWOSV3gDrNnY53ZQmgftvqKb1HxTlJoPfvYV+6c6945efwzSMw44/QerBzK6D9JccvMZuj5ew+0tFyywKnda8lTtKN7wK9bj5yeyamkWdjCQqGdhc5P/vSYfFYWPIRrP3a6ROSdDv0GAW16no2jmpUmVL/5Tgt/TbAR8CHqrpLRCKB1ara4jjHXQMMVdXfu5ZvAvqq6v1u+zQGvgXqAlHAhaq6WETeBn5R1U9c+70PzHAddsJzVsRa/KayiktK2ZtXSFZOIUUlpQQHCcFBQkiQEBwUREiQEHR4WSpYDiJIQKohITml/dWM/20LSc3r8tYNPljaz81y622/ELYvd8rqiJMAmvc7kgBqN/F2tEeoOn0JVkyElMlwcKtzr7njpc6TAa0Gnl752R+pOrdjNrv9nbPSnG0hEdA06cjfOaEPRNQ+8fmqQ3EBrP7S6QuQ8SuE1IKu1zh9AZr09HZ0lXKmLf6rgddV9Uf3laqaJyK3n2Fso4CxqvqaiPQDPhaRLmd4TgBE5E7gToDExMSqOKXxUflFJezOLiArt5A92QXsyXHeV7RuX14hVfGgy9EXDEe/DwkKIigIQoKCjtnn2OWgI8siBAcf2Z6y9QCpO3O4x1XaD63ppX1V516qewLYk+psCw6HpmdB/wedsm5Cb6hVx7vxnogINOrq/Fz4Z9j8s1MFWD0VVnzm3IfucpVzEZCQFFidAktLnIuizW6PT+bscLZF1HEu4nrd7Lw27lEzO9SFhDtVnG7XOdWI5Pedi7ylHzv/nfb+PXS+0nlawAdVpsXfEtiuqvmu5VpAvKqmn+S4fsDzqnqxa/lJAFX9q9s+q3Ba8Bmu5Y3A2Tid+g7vKyKzgOddh53wnBWxFr9/UVUO5hc7yTqnkD05Ba4f1/uyhO7anlNQXOF5YsJDqB8TTlxUGPWjw4mLdl7rx4RTPyqMsJAgikuV0lKluFQpOfxaSkkplJSWHl5fUsl93Pc7slx6zPqKjyk9Kp5aocE8NawjgzrU0NJ+aSnsWnV0os/e7myLiHXukyee7ST6Jj3949n54gJY/53zeOC6mc5z5HVbHukU2KDdyc/hawrzYNuSIx0uM36DQlf3r9hmR/phNO8P9dv77D1z8g/A8glOFWBPqlP67/k7Z1yAeq28Hd0xzrRXfzLQX1ULXcthwM+q2vskx4XgdMQbDGzF6Yh3g6qucttnBvCZqo4VkY7AHKAp0An4lCOd++YAbQE52TkrYom/5ispVfbmHkniZQl9t9t799fCkmOfLhWBepFhRxK4WzJvUC6xx0WFEREa7IVvGkC+vN9pIYHzWJx7AmjQ0XcTQGXlH4A1XzmVgE0/Oo8YNu7uVAG6XA21G3s7wpNThby9cGALHMg88rPfbTl315H9G3Y6+vn5Os28F7unqEL6fOcCYM3XTt+ENhc6VYC2Fzl9BmqAM038y1S1R7l1y1W1eyU+eBjwT5xH7/6nqi+JyAtAsqpOc/Xefw+Ixuno97iqfus69k/AbUAx8JCqzjjeOU8WhyX+mmVR+l7G/bKZXdlHkvze45TYQ4Pl6Na4632Dcom9fnQ4dSND/aMnuz8ozIO/t4R2Q2HIC1AnMbDK3eVl73D6AqyYCNuXAQItz3dKyd4cLri40OmfcCATDmQc/brf9Vp86OhjQmo5CT02wfXTzHmkrlkfiKznne/hLQe3w5IPnQ6B2dud30XSrdDz5hOPGVENzjTxfwe8parTXMsjcJ6rH1zlkXqIJf6aQVUZ8+NG/j5rHbG1QmkRF3lUab1+jCuZu72vHRFSLR3lTBVLnQWfXge/mwJtfOafiuqxZ71TBVgx0en0FhwO7S52LgLaXlR1tzxU4dC+o1vq7i33/RmQsxOnzeUmqqFbYm/m+nFL8pH1AvsiriIlRbBuulMF2PQjBIVC5yucKoCX5gc408TfGhiHU3IXIAO4WVXTqjpQT7HE730HDhXx6OfL+W71ToZ1bcTLV3cjJsIDz1ybmuHrh2H5Z/DHTf5x794TVJ3x5FdMdEaRy93tGi54hGu44HNPfDukpMhpZe53b6lnHJ3oC3OOPiY43EngRyV2t9faTSE0AOdzqEq7U53OgMs+hYKDXpsfoErG6heRaABVzTnZvjWNJX7vStl6gHvGLWb7/nyeGtaRW89pYa14f6YKr3d2OuyNHOftaHxDSTFsmucMErT2aydhxzSBrlc7nSCzt7sldFdyz97u9BtwF1nfLbGXa6nHNoOo+tZary6FuV6dH6AqRu4bDnQGDl8KquoLVRahh1ni9w5VZcKiDJ6btoq4qDDevqEXZzX3n0EwzHHsWAn/Phcufxt63eTtaHxPYZ5TNl75OaTNdo1rAASHOS3yskRe/j577aYQFund2M2xvDQ/wBk9xy8i/wYigUHAf4FrgN+qNELjd/IKi3l6agpTlmzlvLb1eWNkT+pF1cDndU3VS53pvLa9yLtx+KqwSGewmK7XOD3q921yknpUQ/9/EsIfiTiTCDXrDRe/5Jof4H2YdKvX5geozD3+Faraze01GpihqudVT4hnzlr81WvD7hzu/WQJqbuy+cPgtjxwQVuCg6y8GDD+e6EziMudc70diTE1U9n8AIv+C+u/dYYqPms0XPqPKvuIMx25zzWRNXki0gTIAnzgAVTjDV+v2MYfJ60gPDSYD2/tw/ntvPtIi6lmObudsubAJ70diTE1V/n5AZI/cFr/1aQyif8rEakDvAIswXn24z2PRmV8TmFxKX+ZvoaxC9LplViHt2/oRZM6vjmcpTkDad8B6jyeZow5ubotYMifq/UjT5j4RSQImKOq+4HJIvI1EKGqB6olOuMTtu4/xP2fLmHplv3cdk5LnrikA2Ehdi8yIKXOhJjGzgh1xpga6YSJX1VLReQdoKdruQAoqI7AjG/4IXU3D01YSlGJ8q8bezGsq90FCljFhZD2vTM5jT0yZkyNVZlS/xwRuRqYopV96N/4vZJS5Y0563nr+/W0j4/hXzf2olWDaG+HZbxpywJncpZ2Q70diTHmBCqT+O8C/g8oFpF8nNH7VFVrwKTJxhuycgp46LNlzF+/h6t7JfDiFV2oFVYzJqYwXpQ6yxkZrtUAb0dijDmBkyZ+Va2+MQbNmSsthbkvQpdrIL5TlZ9+8ea93DduKXvzCnn56q5cl9TMRuEzziAl62Y4E8+ERXk7GmPMCVRmAJ/zK1qvqj9WfTjmjGX8AvNfc54NvfOHKpsiUlV5/6dN/G3GWprWrcWUe/rTpamXZhQzNU9WmjPQTL/7vB2JMeYkKlPqf8ztfQTQB1gMXOCRiMyZSZkMiDNs6qL3oe+dZ3zK7PwiHp+0ghkpO7ioUzyvXNud2Fo2wY5xUzZanz3GZ0yNV5lS/2XuyyLSDPinxyIyp6+kGFZNdaaDPLQPvn/ReR/d8LRPuWb7Qe4dt4Qte/P407CO/P68llbaN8dKnQUNO0OdRG9HYow5idN52DoT8PzUQubUbfoB8vY49/eHvQpFefDds6d9us+TM7jinZ/JLShm/B1nc8f5rSzpm2Md2g+bF1hr3xgfUZl7/G/hjNYHzoVCD5wR/ExNkzIFwmtDmwudObX7PwA//QN63QzN+1f6NPlFJTz35So+S86gf+s43hjZkwYxNqe6OY4Nc0BL7DE+Y3xEZe7xu89uUwyMV9WfPRSPOV3FBbDmK2eax1DX7MnnP+pM7fnNI3DXjxB88vvy6XtyuWfcEtZsP8gDF7ThoQvb2QQ75sRSZ0GtepBQ4XwgxpgapjKJfxKQr6olACISLCKRqprn2dDMKUmbDQUHoOvVR9aFRcHQv8FnN8JvY07a43pmyg4e+3w5wcHCB6N7M6jD6fcNMAGitMR5gqTtxVX2BIkxxrMqc49/DuA+20otYLZnwjGnbeUkiIyDluUGT+kwHNoMgbl/hYPbKzy0qKSUl75Zzd2fLKZVgyi+fuBcS/qmcjIXOR1J7f6+MT6jMok/QlVzyhZc7yM9F5I5ZYW5zuNUnUYcW84XgWF/h5JC+PZPxxy640A+o8b8wnvzN3Fzv+ZMvLsfCXXtz2sqKXUmBIVAa3u61xhfUZnEnysivcoWROQs4JDnQjKnbN0Mpwd/l2sq3l6vFZz7sPOM/8YfDq/+OW0Pl741n9XbD/LGyB68MKIL4SFWrjWnIHUWJPaDWnW8HYkxppIqk/gfAj4Xkfki8hPwGXC/Z8MypyRlMsQ0cf4BPp5zH4I6zWH6o5QWFfDWnPXc9P6v1I0MY9r95zCiR9Pqi9f4h32bYddq681vjI+pzAA+i0SkA9DetWqdqhZ5NixTaYf2wfrvoO9dEHSC67jQWjDsFfj0Oia98ySv7RjMFT2a8JeruhIZVpk+nsaUs/5b59USvzE+pTLP8d8HjFPVFNdyXREZpar/8nh05uTWfgOlRc4c6CexrFZfsoN6c+m+Twi5+HquHNjDBuQxpy91pnMbqX4bb0dijDkFlSn136Gq+8sWVHUfcIfnQjKnZOUkqNsSmvQ67i6qykcL07n23wt4I/T3RIQIV+1625K+OX2FubBpvrX2jfFBlUn8weKWIUQkGAirzMlFZKiIrBORNBF5ooLtr4vIMtdPqojsd60f5LZ+mYjki8gVrm1jRWST27YelfuqfihntzNMb5ernd77Fe1SUMyDE5bx7JerOL9tA/77h6sIGvAorJnmPPtvzOnY+AOUFNhjfMb4oMrc3J0JfCYi/3Et3wXMONlBrguEd4AhOOP7LxKRaaq6umwfVX3Ybf8HgJ6u9XNxhgZGROoBacC3bqd/TFUnVSJ2/7Z6Kmipk/grkLozm3s+WcymPbk8PrQ9d5/fmqAggf4PwrLxMP0xuPcXCLHheM0pSp0JYTGQWPmhoI0xNUNlWvx/BL4H7nb9rOToAX2Opw+QpqobVbUQmACMOMH+o4DxFay/BphhIwVWIGUyNOgI8Z2O2TR16VZGvP0zBw4VM+73Z3PvwDZO0gcn0Q97BfZuhJ/frOagjc9TdR7ja3MBhFSq+GeMqUFOmvhVtRT4FUjHSeYXAGsqce6mQIbbcqZr3TFEpDnQEucCo7yRHHtB8JKIrHDdKgjM5uqBTNiy8OgheoE9OQU8OWUFD322jK4JsUx/8Fz6tY479vg2g50Bf+a/CvvSqydm4x+2L4ecHXZ/3xgfddxSv4i0w2mFjwL24Dy/j6oO8kAcI4FJZfMBuMXQGOgKzHJb/SSwA6efwRicisQLFcR/J3AnQGKiH84RnjLFee3s9ObfcSCf//y4gfG/baGguJS7BrTisYvaExJ8gmu7i/8C62fDzCdhVEXFFmMqkDoLEGcoaGOMzznRPf61wHzgUlVNAxCRh0+wf3lbgWZuywmudRUZCVQ0g8x1wBfu4waoatmA8wUi8gHwaEUnVNUxOBcGJCUlaUX7+LSUydCkFxnSmH9/sZLPkzMpUeXKnk25Z2BrWjeIPvk5YhNgwOMw+zlYNxPaWwvOVELqTGcmvugG3o7EGHMaTlTqvwrYDswVkfdEZDBwKs9/LQLaikhLEQnDSe7Tyu/kGhyoLrCwgnMcc9/fVQXA9aTBFUDKKcTkH7I2wPZlfFXSj0GvzmNicgbXJCUw79GBvHpt98ol/TJn3wv128OMx6HIRmI2J5G9E7Ytsd78xviw4yZ+VZ2qqiOBDsBcnKF7G4rIuyJy0clOrKrFOEP7zsLpEzBRVVeJyAsicrnbriOBCap6VKtcRFrgVAx+4GjjRGQlTifD+sCLJ4vFn6TuzOab8W9TqsIrWzvxu7Ob8+Pjg/jLlV1pVu80JtcJCYPhr8L+zfDT61UfsPEvNlqfMT5PyuXbE+8sUhe4FrheVQd7LKoqlpSUpMnJyd4O44ykbD3A29+nMXPVduaEP05o7YbUunMWDWKqqG/jpNthzVdw70KIa1015zT+Z8KNsG0pPLzquGNHGGO8T0QWq2pSRdsq8zjfYaq6T1XH+FLS93VLt+zj9rGLuPStn/g5bQ9/7gutZSuJ5/+u6pI+wEUvQnCYU/I/hYtBE0CKC2DDXKfMb0nfGJ9ls7PUUL9uzOKt79P4KW0PdSJDeWRIO27u34LYn/8CEgydrqjaD6zdGAY9CbOegrVfQ8fLqvb8xvel/wRFuVbmN8bHWeKvQVSVn9L28NacNH5L30v96HCeGtaBG/s2Jyo8xGmJp0yGVgMhqn7VB9DnLlg6DmY8Aa0vgLCoqv8M47tSZ0FILWh5vrcjMcacAUv8NYCq8v3aXbz1fRrLMvbTqHYEz1/WiZF9EokIDT6y49bFTie8AX/0TCDBIU5Hvw8ugR9fgQuf98znGN+j6jzG12qAM8WzMcZnWeL3otJSZeaqHbz1fRprth8koW4t/nJlV64+qynhIcHHHpAy2bkP3/FSzwXVvD90HwUL3obuN0CDdp77LOM7dq9zLjrPfcjbkRhjzpAlfi8oLinl6xXbeXtuGmm7cmhVP4pXr+3OiB5NCD3eSHulJc5ofW0vgohYzwY45AVYOx2mPwo3f2kduYzT2gdoa8/vG+PrLPFXo6KSUr5YspV/zUsjPSuP9vExvDmqJ8O7NiY46CTJdfMCZ3z0Lld5PtDohjD4GSfxr5py3Nn/TABJnQWNukJshdNtGGN8iCX+apBfVMLnizP597wNbN1/iC5Na/Pv353FRZ3ij8yYdzIpkyE0qvp6VCfdBks+gll/cqoM4THV87mm5snbCxm/wHmPeDsSY0wVsMTvQYcKS/j0ty2M+XEDOw8W0DOxDi9e0YWB7Rsgp1I+LymC1V9C+0uqr6d9UDAM/we8fyHM+xtc/FL1fK6pedLmgJbaY3zG+AlL/B6QU1DMxws389/5G8nKLeTsVvV4/boe9Gsdd2oJv8zGeXBob/WX3Jv1hl43wy/vQo8bIb5T9X6+qRlSZ0JkfWjSy9uRGGOqgCX+KnQgr4ixC9L538+bOHCoiAHtGnD/BW3o3aLemZ04ZbLToa+NFwZMHPy8M5Tv9Edh9DfW0S/QlBRD2nfQ4VIIOqWBPo0xNZQl/iqQlVPA+z9t4qOFm8kpKGZIp3juH9SG7s3qnPnJiw7Bmq+h8wgIqcIheisrKg4GPwdfPwQrJkL366s/BuM9Gb9C/gGbjc8YP2KJ/wzsOpjPmB83Mu7XLeQXlzCsa2PuG9iGTk1qV92HrP8OCrOhyzVVd85T1esWWPoxfPs0tB/q+ccJTc2ROhOCQqHVIG9HYoypIpb4T8PW/Yf4zw8bmLAog5JSZUT3Jtw7qDVtGnqg53vKZIhqAC3Oq/pzV1ZQEAx7Fd67AOb+BS552XuxmOqVOgtanAMRVXgxa4zxKkv8p+iVWWsZ8+NGAK7ulcA9A1vTPM5DPe0Lsp0WV8+bnOF0valpL+cRv9/GOB39GnfzbjzG8/ZuhD3rIOlWb0dijKlClvhPUXztCEb1SeSuAa1pWsfDY5avmwHF+dDVi2V+d4OfgdVTnY5+t860zl7+LvVb59Xu7xvjVyzxn6Kb+7Wovg9LmQy1EyChT/V95onUqusM5/vlfbD8U+j5O29HZDwpdSbUbwf1Wnk7EmNMFbImW02Vt9cZOKXLlTWrZd39BmjWF7571onR+KeCbEj/yVr7xvihGpRRzFHWfAWlRd7tzV+RoCAY/hoc2gffv+jtaIynbJjr/Pdno/UZ43cs8ddUKZOgXmto3N3bkRyrUVfocyck/w+2LvF2NMYTUmc5j2026+vtSIwxVcwSf02UvQM2zXeG6K2pI+UNesqZxe+bR5wpg43/KC2F9bOgzYUQHOrtaIwxVcwSf020+ktAa/Z0uBGxcNGLsG2JM4uf8R/blkLubivzG+OnLPHXRCsnQXwXaNjB25GcWNdrofm5MOfPkJvl7WhMVUmdCRLktPiNMX7HEn9Ns28zZP5Ws1v7ZURg+KtOD/DZz3k7GlNV1s9y7u1HnuHkUsaYGskSf02z6gvntctV3o2jshp2hLPvccbyz1jk7WjMmTq4DbYvh7YXeTsSY4yHWOKvaVImQdMkqNvC25FU3oA/Qkxj+Ob/rKOfr1tfNlqf3d83xl95NPGLyFARWSciaSLyRAXbXxeRZa6fVBHZ77atxG3bNLf1LUXkV9c5PxORME9+h2q1OxV2rKw5Q/RWVngMXPwX2LECFr3v7WjMmUidBbGJTiXHGOOXPJb4RSQYeAe4BOgEjBKRTu77qOrDqtpDVXsAbwFT3DYfKtumqpe7rX8ZeF1V2wD7gNs99R2qXcpkQKDTFd6O5NR1vhJaDXQG9cnZ5e1ozOkoOgQb5zmj9dXUx0iNMWfMky3+PkCaqm5U1UJgAjDiBPuPAsaf6IQiIsAFwCTXqg8BH8ySFVB1En+Lc6F2Y29Hc+pEnKl7i/Kc4XyN70n/yfn7WZnfGL/mycTfFMhwW850rTuGiDQHWgLfu62OEJFkEflFRMqSexywX1WLT3ZOn7NjJWSt943e/MdTvy30fwCWj4fNC7wdjTlVqTMhNNK5+DTG+K2a0rlvJDBJVd17hjVX1STgBuCfItL6VE4oIne6LhySd+/eXZWxekbKJAgKgU4nKor4gPMfhdhmzoh+JUXejsZUlqpzf7/VIAiN8HY0xhgP8mTi3wo0c1tOcK2ryEjKlflVdavrdSMwD+gJZAF1RKRsOuHjnlNVx6hqkqomNWjQ4HS/Q/VQhZQp0PoC3392OiwKhv4Vdq2G38Z4OxpTWbtWw4EMm43PmADgycS/CGjr6oUfhpPcp5XfSUQ6AHWBhW7r6opIuOt9feAcYLWqKjAXKOv2fgvwpQe/Q/XIXOT8o+vLZX53HS6FNkNg7l/h4HZvR2MqI3Wm82rP7xvj9zyW+F334e8HZgFrgImqukpEXhAR9176I4EJrqRepiOQLCLLcRL931R1tWvbH4H/E5E0nHv+vv/82MpJEBIB7Yd5O5KqIQLD/g4lhfDt096OxlRG6ixo3MM3O5YaY05JyMl3OX2qOh2YXm7ds+WWn6/guAVA1+OccyPOEwP+obTEGa2v7UUQUdvb0VSdeq3g3Ifgh5eh183QaoC3IzLHk5sFGb85AzEZY/xeTencF7jS50PuLv8p87s792Go0xymPwrFhd6OxhxP2neA2v19YwKEJX5vS5kMYdH++Y9uaC0Y9grsSYVf/uXtaE5MFbJ3QNps+PlNmHofrJvp7aiqR+pMiI53Sv3GGL/n0VK/OYniQlg9DToMd5KkP2p3sdN34YeXnaGIYxO8HREU5sKutbBrFex0+zm098g+weGw5iu471f/vu9dUgRpc5zHSIOsHWBMILDE700bvof8/f5Z5nc39G/wTl+Y+SRc/3H1fW5pCezd5Erwq2FnivPY2t5NgKsvaWiUMy59x8sgvjM07OS8HtoH7/Z3xiMYOc5/h7DdshAKDtpofcYEEEv83pQyGWrVdQZN8Wd1m8P5jzjj+KfNhjYXVv1n5O450nIva8nvWgvFh5ztEgT1WkOjrtBtpJPc4ztBnRYVt3Qj68GgP8F3zzidL31lmuRTlToLgsOceRaMMQHBEr+3FObBuulOaz/EfyYYPK7+D8Ky8TD9cbh3IYSEn955ivJh91qn5e5eps91mxgoqoHTck+6zUnu8Z2hQYdTv51y9r2wagpMf8xJjL4+uFJFUmdCi/MgPNrbkRhjqoklfm9ZPwsKc/y/zF8mJNzp6PfJVU7nuQGPnXj/0lLYv9mV4N3K9FlpoKWuc0Y4Cb3tkKPL9NENqybm4BAY8Q7853yY+QRc5WcjEe5Jc36ffe7ydiTGmGpkid9bUiY7PakDaUKUNoOdTmTzX4Vu1zm3AADy9h5J8IfL9GucC6MydVtAfBdnyuL4zs5PvVYQFOzZmOM7w3mPOJ0Tu1wD7fxoZLv1s5xXf/pOxpiTssTvDfkHIPVbSLrV84mrprn4L7B+Nky8CaIaOkk+e9uR7bXqQsPO0ONGV5m+i9Oq92Yp+rxHnKcvvn4I7v3FfwZaSp0JDTo6F1XGmIBhid8b1k6HkoLAKfO7i02Awc/Cd89C/VJoeZ6rTO/qbBfTuOb1oA8JhxFvw/tDYPZzcOnr3o7ozOUfcKZO7ne/tyMxxlQzS/zekDIZYhMhobe3I/GOs++GvnfVvAR/IglJTme/hW87F2y+fotmw/dQWmyP8RkTgGzEjuqWmwUb5zqPh/lS4qtqvvjdB/3JKYtPe8B5KsOXpc5ybqsE6sWnMQHMEn91W/Ol09Lqes3J9zU1S1gkXP4W7N0I8/7q7WhOX2kJrP/WmTo52Ip+xgQaS/zVLWUK1G/ndFozvqfl+XDWaKfkv3Wxt6M5PVsXQ16Wf84PYYw5KUv8Tdd8YgAAEFdJREFU1engNkj/yblH7IulbuMY8gJEN4IvH/DNWQdTZ4IEO49XGmMCjiX+6rRqKqCB2Zvfn0TEwqX/cMYc+MkHe/inzoLEfs49fmNMwLHEX53+f3t3HmVlfd9x/P1xBnBQQTYRWQQDLmhckgmJMdVUosFoICfpUTxJY3PSpEtQa3oSTdrENHuz1MRoc2pTl1SPnhwgERcYqZhgGqCgUhlEB0TCEpChiAY3YPj2j9+DXoGBWe5zH+7cz+ucOfe5v/vc5/nehxm+9/mtzdPh2NNh8LiiI7HuOumiNKHP/O+niYeqxbZ1aRZEV/Ob1Swn/krZ+lxqW/Xdfs9x0T+nyXxmTUsd5qrBG7P1eRifWa1y4q+U5TPTY09d5a0WHTEYLvpe+kK38KdFR9MxLU0wYIxrncxqmBN/pSybASPfDUePKjoSK6fTPgYnXpSWHN66uuhoDmzHK/Dc/HS3786lZjXLib8SNq9IHcFczd/zSKmjX10vmHUVRBQdUfuemw+7XnP7vlmNc+KvhOaZoMPSynLW8/Q7Di78Bqx5FB6/o+ho2tcyB3ofCcefU3QkZlYgJ/68RaTe/KP/BI4aWnQ0lpd3XJH+jR/6Cry4oeho9hWR2vffdj7U9y46GjMrkBN/3jYuTW2/nqK3Z5Ng8o3QthMe+PyhV+W/aVla/ti9+c1qnhN/3ppnwGG94ORLio7E8jbwBJj4lVSl3jyj6GjeqiUbxjfugmLjMLPCOfHnaffu1L4/diL0HVh0NFYJ7/5rGN4Is78IL28pOpo3tcyB4e+EI48pOhIzK1iuiV/SJEnPSFol6br9vH6DpKXZT4ukbVn5mZIWSFou6UlJl5W853ZJz5W878w8P0O3rFsEL21IM7xZbTisDqbcBK+9BLOvLTqaZPvmNNeAq/nNDMhtTU5JdcDNwAXAemCxpFkR8cb8phFxTcn+VwJnZU9fAT4ZESslHQc8JqkpIrZlr38hIqbnFXvZNE+H+oY0vavVjmNOgXO/AL/+durbUfS//8q5QHgYn5kB+d7xTwBWRcTqiNgB3ANMOcD+lwN3A0RES0SszLb/AGwGhuQYa/m17UqL8pz4QehzZNHRWKW97xo45lS4/xp47cViY2mZA0cNS+tEmFnNyzPxDwfWlTxfn5XtQ9LxwBhg3n5emwD0Bp4tKf5W1gRwg6Q+5Qu5jNbMh1e2uDd/rarvnar8tz+fhvgVZdcOeHZe+gLq2frMjEOnc99UYHpEvGWlE0nDgP8EPhURu7PiLwEnA+8CBgL7bUiV9FlJSyQtaW1tzS/y9iybAX36wVj3oq5Zw98BZ09Lk/qs/k0xMfz+v2HHdrfvm9kb8kz8G4CRJc9HZGX7M5Wsmn8PSf2AB4B/iIiFe8ojYmMkrwO3kZoU9hERt0REY0Q0DhlS4VaCXa/DivvSEL5eh1f23HZo+dMvp2F+910FO16u/PlbmqD+cBhzXuXPbWaHpDwT/2JgnKQxknqTkvusvXeSdDIwAFhQUtYb+CXw87078WW1AEgS8BGgObdP0FWrHobXX/Tc/Aa9GmDyTfDCGpj3rcqeOwJaZsOYc6F338qe28wOWbkl/ojYBUwDmoAVwC8iYrmkr0uaXLLrVOCeiLdMdXYpcC7wF/sZtneXpGXAMmAw8M28PkOXNU+HhoFwgu+yDBh9DjR+Ghb+K6xbXLnzblmZvnC4N7+ZlchtOB9ARDwIPLhX2Vf3ev61/bzvTuDOdo55fhlDLL8dL8Mzs+GMqWnFNjOAD3wtVbvPmgZ/NR/qK9AntWVOehznxG9mbzpUOvf1HM/Mhp2vuJrf3urwfnDJDdD6NDz6w8qcs6UJhp4GR488+L5mVjOc+MuteWYaMz3q7KIjsUPNiRfC6ZelxL8p564pr74Aaxe4mt/M9uHEX06vboNVc+HUj6apW832Num70DAgVfm37crvPKsehmjzMD4z24cTfzk9fT+07XA1v7Wv70D40PfhD0/AwpvzO09LE/QdlBbmMTMr4cRfTs0zYMDoNHGLWXvGfyTN8fDIt2HLqvIfv21Xqnkad6FrnsxsH0785bK9Nc3OdtrHPDWqHZgEF/8Q6vqkiX127z74ezpj/eLUxu/2fTPbDyf+cnnqV6lN1dX81hFHHQsf/FaaUvexW8t77JY5cFg9vO3QHvlqZsVw4i+X5pkw5BQYemrRkVi1OOsTcML7Ye71sG3dwfbuuJYmOP69cHj/8h3TzHoMJ/5yeHE9rP2d7/atcyT48I8hdqfle98yeWUXvbAGWle4N7+ZtcuJvxyW/zI9nvbRYuOw6jNgNEy8PnXGe/IX3T9ey0Pp0YnfzNrhxF8OzTPguLNg0NuKjsSq0YTPwIgJMOda2L65e8dqmQODxvp30cza5cTfXf/3bBqT7Wp+66rD6mDKTWmdh9lf7PpxXt8Oax713b6ZHZATf3c1z0yPp7qa37phyElw3rWp2WjF/V07xupfpwmkPIzPzA7Aib+7mmfAqPdC/+FFR2LV7pyr4di3wwOfT+PwO6tlDvTp53UizOyAnPi74/nlqQe1O/VZOdT1gsk3wctb4KF/7Nx7d++GlQ/B2IleDtrMDsiJvzuaZ4Dq0hSsZuVw3JlwzlXwxJ3w7LyOv2/jUtj+vNv3zeygnPi7KiIl/hPOgyOHFB2N9STnXZd65t93deqw1xEtTYBg7AW5hmZm1c+Jv6s2PJ4mS3Fvfiu3XoenKv9t62DeNzr2npVNMHICHDEo39jMrOo58XdV8wyo651WWTMrt+PPTuP7F/0brF104H3/uCkNKXVvfjPrACf+rtjdBstnpmrVhqOLjsZ6qolfhf4jYNY02Pla+/ut9Gx9ZtZxTvxdsXYB/HGje/Nbvvocleby39IC87/X/n4tTdB/JBwzvnKxmVnVcuLviuYZ0KsvnHRR0ZFYTzd2Ipz5cfjtj2Djk/u+vvM1ePaRVM0vVT4+M6s6Tvyd1bYTlv8qJf3eRxQdjdWCC78JfQfBvZ9Lv3+lfv9b2Pmyq/nNrMOc+Dtr9W/g1a1w2p8VHYnVir4D4eIfwqYn4Xc/eetrLU1Q3wCj31dMbGZWdZz4O6t5BvTpn6pgzSpl/GQ4ZTL8+ruwZWUqi0jT9J7wfujVUGR0ZlZFnPg7612fhot/APV9io7Eas2HfpAS/L3T0hS9rU/DtrUexmdmnZJr4pc0SdIzklZJum4/r98gaWn20yJpW8lrV0hamf1cUVL+TknLsmPeKFW4R9OIRjj90oqe0gyAo4bCpO/AuoWw+Gfpbh+c+M2sU+rzOrCkOuBm4AJgPbBY0qyIeGrPPhFxTcn+VwJnZdsDgeuBRiCAx7L3vgD8FPgMsAh4EJgEzM7rc5gdUs64HJZNh//6Whrjf+zp0O+4oqMysyqS5x3/BGBVRKyOiB3APcCUA+x/OXB3tv1BYG5EbM2S/VxgkqRhQL+IWBgRAfwc8Ao5Vjsk+PCP0uOWZ9yb38w6Lc/EPxxYV/J8fVa2D0nHA2OAPcuRtffe4dn2QY9p1mMdPQou+Ke0fYqnjDazzsmtqr+TpgLTI6KtXAeU9FngswCjRo0q12HNDg3v+ks46WLoN6zoSMysyuR5x78BGFnyfERWtj9TebOa/0Dv3ZBtH/SYEXFLRDRGROOQIV4213ogJ30z64I8E/9iYJykMZJ6k5L7rL13knQyMABYUFLcBFwoaYCkAcCFQFNEbARekvSerDf/J4F7c/wMZmZmPUpuVf0RsUvSNFISrwNujYjlkr4OLImIPV8CpgL3ZJ319rx3q6RvkL48AHw9IrZm238L3A40kHrzu0e/mZlZB6kk3/ZYjY2NsWTJkqLDMDMzqwhJj0VE4/5e88x9ZmZmNcSJ38zMrIY48ZuZmdUQJ34zM7Ma4sRvZmZWQ5z4zczMaogTv5mZWQ2piXH8klqB35fxkIOBLWU8Xi3yNew+X8Py8HXsPl/D7iv3NTw+IvY7X31NJP5yk7SkvYkRrGN8DbvP17A8fB27z9ew+yp5DV3Vb2ZmVkOc+M3MzGqIE3/X3FJ0AD2Ar2H3+RqWh69j9/kadl/FrqHb+M3MzGqI7/jNzMxqiBN/J0maJOkZSaskXVd0PNVG0khJj0h6StJySVcXHVO1klQn6QlJ9xcdSzWSdLSk6ZKelrRC0tlFx1RtJF2T/R03S7pb0uFFx1QNJN0qabOk5pKygZLmSlqZPQ7I6/xO/J0gqQ64GbgIGA9cLml8sVFVnV3A30fEeOA9wOd8DbvsamBF0UFUsR8DcyLiZOAMfC07RdJw4CqgMSJOA+qAqcVGVTVuBybtVXYd8HBEjAMezp7nwom/cyYAqyJidUTsAO4BphQcU1WJiI0R8Xi2/UfSf7bDi42q+kgaAVwM/KzoWKqRpP7AucB/AETEjojYVmxUVakeaJBUD/QF/lBwPFUhIuYDW/cqngLckW3fAXwkr/M78XfOcGBdyfP1OGl1maTRwFnAomIjqUo/Ar4I7C46kCo1BmgFbsuaS34m6Yiig6omEbEB+AGwFtgIvBgRDxUbVVUbGhEbs+1NwNC8TuTEb4WQdCQwA/i7iHip6HiqiaRLgM0R8VjRsVSxeuAdwE8j4izgZXKsWu2JsjboKaQvUccBR0j6RLFR9QyRhtvlNuTOib9zNgAjS56PyMqsEyT1IiX9uyJiZtHxVKFzgMmS1pCam86XdGexIVWd9cD6iNhT2zSd9EXAOu4DwHMR0RoRO4GZwHsLjqmaPS9pGED2uDmvEznxd85iYJykMZJ6kzqyzCo4pqoiSaR21RUR8S9Fx1ONIuJLETEiIkaTfgfnRYTvtDohIjYB6ySdlBVNBJ4qMKRqtBZ4j6S+2d/1RNxBsjtmAVdk21cA9+Z1ovq8DtwTRcQuSdOAJlIP1lsjYnnBYVWbc4A/B5ZJWpqVfTkiHiwwJqtNVwJ3ZV/iVwOfKjieqhIRiyRNBx4njdZ5As/g1yGS7gbeDwyWtB64Hvgu8AtJnyatJntpbuf3zH1mZma1w1X9ZmZmNcSJ38zMrIY48ZuZmdUQJ34zM7Ma4sRvZmZWQ5z4zeygJLVJWlryU7ZZ7iSNLl2lzMzy5XH8ZtYRr0bEmUUHYWbd5zt+M+sySWskfU/SMkn/I2lsVj5a0jxJT0p6WNKorHyopF9K+t/sZ88Ur3WS/j1b2/0hSQ2FfSizHs6J38w6omGvqv7LSl57MSLeDtxEWjUQ4CfAHRFxOnAXcGNWfiPwm4g4gzQ3/p6ZL8cBN0fEqcA24GM5fx6zmuWZ+8zsoCRtj4gj91O+Bjg/IlZniy9tiohBkrYAwyJiZ1a+MSIGS2oFRkTE6yXHGA3MjYhx2fNrgV4R8c38P5lZ7fEdv5l1V7Sz3Rmvl2y34f5HZrlx4jez7rqs5HFBtv070sqBAB8HHs22Hwb+BkBSnaT+lQrSzBJ/qzazjmgoWU0RYE5E7BnSN0DSk6S79suzsiuB2yR9AWjlzZXvrgZuyVYgayN9CdiYe/Rm9ga38ZtZl2Vt/I0RsaXoWMysY1zVb2ZmVkN8x29mZlZDfMdvZmZWQ5z4zczMaogTv5mZWQ1x4jczM6shTvxmZmY1xInfzMyshvw/ZkqchJkm4C8AAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 576x360 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "n5smD3JyzlDn"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}