pytorch_unet_resnet18_colab.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "pytorch-unet-resnet18-colab.ipynb",
      "provenance": [],
      "authorship_tag": "ABX9TyOdlE+t0YU4MFBwMIqB8ltC",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "26c23e86b3144131b1a461204794c75c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_da049760188f4fcbbb489bd6112c4465",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a56301de9c984f33afb92133a9b695ea",
              "IPY_MODEL_a4a0557cdac841dabc49bb820eee9dda"
            ]
          }
        },
        "da049760188f4fcbbb489bd6112c4465": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a56301de9c984f33afb92133a9b695ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_7f8d3d80a279490a88db38957610eb85",
            "_dom_classes": [],
            "description": "100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 46827520,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 46827520,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7c346e79cfd54349b4c1741176b75d9d"
          }
        },
        "a4a0557cdac841dabc49bb820eee9dda": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ddfaa576cef74773a11e1e97aa366530",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 44.7M/44.7M [00:54&lt;00:00, 862kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_096d1eecb02c4ef9a6ecee295d865419"
          }
        },
        "7f8d3d80a279490a88db38957610eb85": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7c346e79cfd54349b4c1741176b75d9d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ddfaa576cef74773a11e1e97aa366530": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "096d1eecb02c4ef9a6ecee295d865419": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/usuyama/pytorch-unet/blob/master/pytorch_unet_resnet18_colab.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4lcY1ziTLblo",
        "colab_type": "text"
      },
      "source": [
        "## pytorch-uent\n",
        "\n",
        "https://github.com/usuyama/pytorch-unet"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yUvckFGU-4HE",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 137
        },
        "outputId": "b7c6934b-b2a7-4654-aa03-955576b2fb29"
      },
      "source": [
        "import os\n",
        "\n",
        "if not os.path.exists(\"pytorch_unet.py\"):\n",
        "  if not os.path.exists(\"pytorch_unet\"):\n",
        "    !git clone https://github.com/usuyama/pytorch-unet.git\n",
        "\n",
        "  %cd pytorch-unet"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Cloning into 'pytorch-unet'...\n",
            "remote: Enumerating objects: 9, done.\u001b[K\n",
            "remote: Counting objects: 100% (9/9), done.\u001b[K\n",
            "remote: Compressing objects: 100% (9/9), done.\u001b[K\n",
            "remote: Total 64 (delta 4), reused 0 (delta 0), pack-reused 55\u001b[K\n",
            "Unpacking objects: 100% (64/64), done.\n",
            "/content/pytorch-unet\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EAx84Zg1_RnV",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 86
        },
        "outputId": "b96f12d2-dd61-4d5e-e4be-9446fb645b76"
      },
      "source": [
        "!ls"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "helper.py  pytorch_fcn.ipynb\t\tpytorch_unet_resnet18_colab.ipynb\n",
            "images\t   pytorch_resnet18_unet.ipynb\tREADME.md\n",
            "LICENSE    pytorch_unet.ipynb\t\tsimulation.py\n",
            "loss.py    pytorch_unet.py\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "N90-BlegJZfs",
        "colab_type": "text"
      },
      "source": [
        "## Enabling GPU on Colab\n",
        "\n",
        "Need to enable GPU from Notebook settings\n",
        "\n",
        "- Navigate to Edit-Notebook settings menu\n",
        "- Select GPU from the Hardware Accelerator dropdown list\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HCitpQdkJNdI",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        },
        "outputId": "541fd863-67b2-460a-dc18-212606460569"
      },
      "source": [
        "import torch\n",
        "\n",
        "if not torch.cuda.is_available():\n",
        "  raise Exception(\"GPU not availalbe. CPU training will be too slow.\")\n",
        "\n",
        "print(\"device name\", torch.cuda.get_device_name(0))"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "device name Tesla P100-PCIE-16GB\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v8nZ6_mKMsJs",
        "colab_type": "text"
      },
      "source": [
        "## Synthetic images for demo training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6qt0VHVZ_53z",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        },
        "outputId": "f065f339-e427-4778-dcb6-9ce047f1958b"
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "import helper\n",
        "import simulation\n",
        "\n",
        "# Generate some random images\n",
        "input_images, target_masks = simulation.generate_random_data(192, 192, count=3)\n",
        "\n",
        "print(\"input_images shape and range\", input_images.shape, input_images.min(), input_images.max())\n",
        "print(\"target_masks shape and range\", target_masks.shape, target_masks.min(), target_masks.max())\n",
        "\n",
        "# Change channel-order and make 3 channels for matplot\n",
        "input_images_rgb = [x.astype(np.uint8) for x in input_images]\n",
        "\n",
        "# Map each channel (i.e. class) to each color\n",
        "target_masks_rgb = [helper.masks_to_colorimg(x) for x in target_masks]"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "input_images shape and range (3, 192, 192, 3) 0 255\n",
            "target_masks shape and range (3, 6, 192, 192) 0.0 1.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "t16ni593BSUE",
        "colab_type": "text"
      },
      "source": [
        "# Left: Input image (black and white), Right: Target mask (6ch)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dzjh6C1HBTCb",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 704
        },
        "outputId": "b97c977d-15eb-4fc5-8e8e-8e1b38f94712"
      },
      "source": [
        "helper.plot_side_by_side([input_images_rgb, target_masks_rgb])"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAdsAAAKvCAYAAAAiIWV+AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3df6xlZX3v8c/nDOIfQAPKzmQcmPKjI0YaOcLJ1FAhKGKBFEcgsTNpdLSkZ8iFpL1t0otyU81NbIyVkhiv6KFMgBs7YAWU2mkrl2vk0kDljA7jgCIzCGGGceYIN+KvYJnzvX+ctXFx2OfsH2s9Z/3Y71eyc/Z+9lp7f9fMefbnPGut/SxHhAAAQDoTVRcAAEDbEbYAACRG2AIAkBhhCwBAYoQtAACJEbYAACSWLGxtX2z7Cdt7bV+X6n0AAKg7p/iere1Vkn4o6SJJ+yU9ImlzRDxe+psBAFBzqUa2GyTtjYinIuLXku6QtDHRewEAUGtHJXrdtZKezT3eL+n3llrYNtNYYZz9JCI6VRdRlhNPPDFOOeWUqssAKrFz586e/TlV2PZle1rSdFXvD9TIM1UXUFS+P69bt06zs7MVVwRUw3bP/pxqN/IBSSfnHp+Utb0iImYiYioiphLVAGCF5Ptzp9OaQTpQmlRh+4ik9bZPtX20pE2S7k30XgAA1FqS3cgR8bLtayX9m6RVkrZFxGMp3gsAgLpLdsw2InZI2pHq9QEAaApmkAIAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEissovHA0DVIqL017Rd+mui+QjbIc3Pz2tiYqL0nwBWVkTo3N/5m9Jf96F915f+mmg+PuWH1A3Gsn8CANqLT/ohzc/PJ/kJAGivkcPW9sm2v2n7cduP2f6zrP0Ttg/Y3pXdLi2v3OoxsgUADKvIMduXJf1lRHzH9nGSdtq+L3vuxoj4TPHy6odjtgCAYY0cthFxUNLB7P7PbH9f0tqyCqsrRrYAgGGV8klv+xRJb5f0H1nTtbZ3295m+4Qy3qMuOGYLABhW4bC1faykuyT9eUS8KOkmSadLmtTCyPeGJdabtj1re7ZoDSuJkS3wWvn+PDc3V3U5QO0U+qS3/TotBO2XIuJuSYqIQxFxJCLmJd0saUOvdSNiJiKmImKqSA0rjZEt8Fr5/tzpdKouB6idImcjW9Itkr4fEX+Xa1+TW+xySXtGL69+GNkCAIZV5Gzk35f0QUnfs70ra/uYpM22JyWFpKclbS1UYc1wNjIAYFhFzkZ+UFKvSUB3jF5O/TGyBQAMi0/6IXHMFgAwLMJ2SIxsAQDD4pN+SIxsAQDDImyHxMgWADAsPumHxMgWADAswnZIjGwBAMPik35IjGwBAMMibIfEyBYAMCw+6YfEyBYAMKwi0zWOJUa2QDvY1kP7rq+6DIwJPukBAEiMsAUAIDHCFgCAxAhbAK0XEVWXgDFH2C6Bs4SBdogITcxcRuCiUoTtEmwTuECLELioUuGwtf207e/Z3mV7Nmt7g+37bD+Z/TyheKkrj8AF2oXARVXKGtm+KyImI2Iqe3ydpPsjYr2k+7PHjUTgAu1C4KIKqXYjb5R0W3b/NknvT/Q+K4LABdqFwMVKKyNsQ9I3bO+0PZ21rY6Ig9n9H0taXcL7VIrABdqFwMVKKiNs3xkRZ0u6RNI1ts/PPxkLv82v+Y22PW17tnuctwkIXKC3fH+em5urupyBEbhYKYXDNiIOZD8PS7pH0gZJh2yvkaTs5+Ee681ExFTuOG8jELjAa+X7c6fTqbqcoRC4WAmFwtb2MbaP696X9F5JeyTdK2lLttgWSV8r8j51Q+AC7ULgIrWiI9vVkh60/aikb0v654j4V0mfknSR7SclvSd73CoELtAuBC5SKnSJvYh4StJZPdqfl3Rhkddugm7gcpk8oB0mZi7T/PQ/yXbVpaBlSImCGOEC7cIIFykQtiUgcIF2IXBRNsK2JAQu0C4ELspE2C4hIoa+SVwtCGiTiZnLqi4BLVHoBKk246QnoB1sK7Z+veoyMOZIFAAAEiNsAQBIjLAFACAxwhYAgMQIWwAAEiNsAQBIjLAFACAxwhYAgMQIWwAAEiNsAQCNk58mtwkIW4yNiGDuaqAFIkK3P3iubn/w3KpLGdjIcyPbPkPSnbmm0yT9taTjJf2ppLms/WMRsWPkCgEAaLiRwzYinpA0KUm2V0k6IOkeSR+RdGNEfKaUCgEAaLiydiNfKGlfRDxT0usBANAaZYXtJknbc4+vtb3b9jbbJ5T0HgAANFLhsLV9tKT3SfrHrOkmSadrYRfzQUk3LLHetO1Z27NFawBQrXx/npub678CMGbKGNleIuk7EXFIkiLiUEQciYh5STdL2tBrpYiYiYipiJgqoQYAFcr3506nU3U5QO2UEbablduFbHtN7rnLJe0p4T0AAGiskc9GliTbx0i6SNLWXPOnbU9KCklPL3oOAIBlDTNZxSDL2i5STikKhW1E/ELSGxe1fbBQRcAIBu2ctvsuGxGamGC+F6AK3QkrBjXIslvOe6hISaUoFLZAXQzz122TpngD0A6ELVphkJFody5VRq1AfdnuOxLNj37rMGodBJ86AAAkxsgWAAoY5rBEHU7UKdty29/G7R0VYVuBQTtnW3d5LrX9bd1etFP393jr3c8NvM4Xr3iTpHaEUHf7f/7oqiWXOfasI5Lasb1FEbYraNgTc7pnzrYlhPptf9u2F+0VEUOFbFd3nS9e8aZGB1BELBuyXd1ljj3rSKO3twx8oq2QImfA2m78dViH3dXW9O1Fe40atHlb736usWfFDxq0ecMu30aE7QpYbrfpUrfFCCCgemUEbVeTA3cU47StvbAbObFev2DdtuV2lXaDNb/rpRu4TdvFOkona+q2or36Be3MlWuHXm/r3c81apfyKKParp8/ukrHTY7vgIFPsoSWCtqJiYm+IdJdZvFrMMId3bj/ZY3RLReYM1euXTJopYU+O3Pl2ldOjlps3Ea444qwTWS5oB0GgVueQf7IARbrF7SDsr1s4GJw3YkvmjKhhUTYrpgiZ9j2ClwA1RomaLuWC1z6eLsRtgks7jRlfJVl8fqMboH0lhrVjhK0XUsFLqPbdiNsG4S/fAGgmQjbxMqcoKGpxxv5IwFtUmRU27Xc7mS0UzM/vdEoo/yRwCxSQP3YfmUKxmGN89d+pAHD1vY224dt78m1vcH2fbafzH6ekLXb9mdt77W92/bZqYpHczC6BepluUl1+k2yg+ENOqnFrZI+J+n2XNt1ku6PiE/Zvi57/N8kXSJpfXb7PUk3ZT/RcEUuoDAxMdFzoo6l3oNRLZBOROiRd7574OU3/Ps3X7nfHd0OM7nFuI9qpQFHthHxgKQXFjVvlHRbdv82Se/Ptd8eCx6WdLztNWUUi2bLT9Sx1I3vwgL1Z1vHTc733aV83OQ8QZspMl3j6og4mN3/saTV2f21kp7NLbc/azsoQIxagbbohi76K+VTLxb2/Q21Y9/2tO1Z27Nl1ACgOvn+PDc3V3U5SXEME6MoEraHuruHs5+Hs/YDkk7OLXdS1vYqETETEVMRMVWghtorc/IJOjnqKt+fO51O1eUkVcbkE2VePQjNUCRs75W0Jbu/RdLXcu0fys5Kfoekn+Z2N48FQhFoN/o4hjXoV3+2S3pI0hm299u+StKnJF1k+0lJ78keS9IOSU9J2ivpZkn/pfSqay7F1IoppoAEsLzlplYcNXBTTAGJ+hvoBKmI2LzEUxf2WDYkXVOkqDaIiNKuRctf0UD9jHItWnYfjy+GRomUdWm8si7VB2A0/S6NN8jED91lyrhUH5qpyFd/0Ed3IofFI9z8d0qXslTnJWiBldcN3F5h2W1bbq7j5UazBO14IGwT6xW40m9CdxhVBy27szHOlgtcabSzlAna8UHYroClAncYVQetxGQUQL/AHUaVQWv7VVMwIj0+PVdIfprCYQyyyxnAyinj8niMaMcPI9sV1A3MYU6SImSB+rGtmSvXDn12MSE7vgjbChCgQDt0Qxfoh099AAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAAS6xu2trfZPmx7T67tb23/wPZu2/fYPj5rP8X2r2zvym5fSFk8AABNMMjI9lZJFy9qu0/S70bE2yT9UNJHc8/ti4jJ7HZ1OWUCANBcfcM2Ih6Q9MKitm9ExMvZw4clnZSgNgAAWqGMY7Z/Iulfco9Ptf1d29+yfV4Jrw8AQKMVuuqP7eslvSzpS1nTQUnrIuJ52+dI+qrtMyPixR7rTkuaLvL+AOoh35/XrVtXcTVA/Yw8srX9YUl/KOmPI7siekS8FBHPZ/d3Ston6c291o+ImYiYioipUWsAUA/5/tzpdKouB6idkcLW9sWS/krS+yLil7n2ju1V2f3TJK2X9FQZhQIA0FR9dyPb3i7pAkkn2t4v6eNaOPv49ZLusy1JD2dnHp8v6X/Y/k9J85KujogXer4wAABjom/YRsTmHs23LLHsXZLuKloUAABtwgxSAAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAifUNW9vbbB+2vSfX9gnbB2zvym6X5p77qO29tp+w/QepCgcAoCkGGdneKuniHu03RsRkdtshSbbfKmmTpDOzdT5ve1VZxQIA0ER9wzYiHpD0woCvt1HSHRHxUkT8SNJeSRsK1AcAQOMVOWZ7re3d2W7mE7K2tZKezS2zP2sDAGBsjRq2N0k6XdKkpIOSbhj2BWxP2561PTtiDQBqIt+f5+bmqi4HqJ2RwjYiDkXEkYiYl3SzfrOr+ICkk3OLnpS19XqNmYiYioipUWoAUB/5/tzpdKouB6idkcLW9prcw8sldc9UvlfSJtuvt32qpPWSvl2sRAAAmu2ofgvY3i7pAkkn2t4v6eOSLrA9KSkkPS1pqyRFxGO2vyzpcUkvS7omIo6kKR0AgGboG7YRsblH8y3LLP9JSZ8sUhQAAG3CDFIAACRG2AIAkBhhCwBAYoQtAACJEbYAACRG2AIAkBhhCwBAYoQtAACJEbYAACRG2AIAkBhhCwBAYoQtAACJEbYAACRG2AIAkBhhCwBAYoQtAACJ9Q1b29tsH7a9J9d2p+1d2e1p27uy9lNs/yr33BdSFg8AQBMcNcAyt0r6nKTbuw0R8Ufd+7ZvkPTT3PL7ImKyrAIBAGi6vmEbEQ/YPqXXc7Yt6QOS3l1uWQAAtEfRY7bnSToUEU/m2k61/V3b37J9XsHXBwCg8QbZjbyczZK25x4flLQuIp63fY6kr9o+MyJeXLyi7WlJ0wXfH0AN5PvzunXrKq4GqJ+RR7a2j5J0haQ7u20R8VJEPJ/d3ylpn6Q391o/ImYiYioipkatAUA95Ptzp9OpuhygdorsRn6PpB9ExP5ug+2O7VXZ/dMkrZf0VLESAQBotkG++rNd0kOSzrC93/ZV2VOb9OpdyJJ0vqTd2VeBviLp6oh4ocyCAQBomkHORt68RPuHe7TdJemu4mUBANAezCAFAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJOaIqLoG2Z6T9AtJP6m6lhKcKLajTpqwHb8dEa25Lp3tn0l6ouo6StCE351BsB0rq2d/rkXYSpLt2TZc25btqJe2bEeTtOXfnO2ol6ZvB7uRAQBIjLAFACCxOoXtTNUFlITtqJe2bEeTtOXfnO2ol0ZvR22O2QIA0FZ1GtkCANBKhC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBiycLW9sW2n7C91/Z1qd4HAIC6c0SU/6L2Kkk/lHSRpP2SHpG0OSIeL/3NAACouVQj2w2S9kbEUxHxa0l3SNqY6L0AAKi1VGG7VtKzucf7szYAAMbOUVW9se1pSdPZw3OqqgOogZ9ERKfqIorI9+djjjnmnLe85S0VVwRUY+fOnT37c6qwPSDp5Nzjk7K2V0TEjKQZSbJd/oFjoDmeqbqAovL9eWpqKmZnZyuuCKiG7Z79OdVu5Eckrbd9qu2jJW2SdG+i9wIAoNaSjGwj4mXb10r6N0mrJG2LiMdSvBcAAHWX7JhtROyQtCPV6wMA0BTMIAUAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGLJrvoDAGi2iOi7jO0VqKT5CFuUql/njAhNTLBDBaizbj/eevdzfZf94hVvkkTo9kPYohSD/AUsLXTIiCB0gZqKiIFCtqu77BeveBOBu4yRP+1sn2z7m7Yft/2Y7T/L2j9h+4DtXdnt0vLKRR31CtpuoOZvebY1Pz+/UiUCGMCwQZu39e7nBv6jexwVGdm+LOkvI+I7to+TtNP2fdlzN0bEZ4qXh7pb3Lm6j3uNWrvh2v3rtxu4jHCB6i0XtDNXrh1o2a13P8cIdwkjh21EHJR0MLv/M9vfl7R2+bXQJr2Cdrng7D43Pz9P4AI1slR4Lg7ZLtuauXJtz/UI3N5K+YSzfYqkt0v6j6zpWtu7bW+zfUIZ74F6GTZo8yYmJl61PruUgeoMG7R5tl85QSpv1F3RbVY4bG0fK+kuSX8eES9KuknS6ZImtTDyvWGJ9aZtz9qeLVoDqjXKyU6LAxfNlu/Pc3NzVZeDggYJ2q6lApf+/WqFwtb267QQtF+KiLslKSIORcSRiJiXdLOkDb3WjYiZiJiKiKkiNWDl5TtRkbOK8+sxum22fH/udDpVl4MB9RrVDhO0Xb0Cl9HtqxU5G9mSbpH0/Yj4u1z7mtxil0vaM3p5aDv++gUwDoqMbH9f0gclvXvR13w+bft7tndLepek/1pGoaifMr4ry4lRQH2MMqrtWmp3MhYUORv5QUm9TjfbMXo5AAC0D8MKAAASI2wBAEiMsAUASOKExZQIW4ysjK/r0LmB+ijydZ0i8yqPA8IWQyMggfaif6dB2GJoZU1GUdbkGABGs9RkFMMGblmTY7QZn24YSdG5jfnrGaivYQKX3ceDIWwxkiIXEyhyEQMA5VruYgK9rkXd1X1u1IsYjJsi17PFmJuYmHjN5fK6HbBXeC51kXmCFqhWN3B7XS5P0lBX9iFoeyNsUcjiwJV+E7r9ELTAcEY5/DLodWWXClxp8LOUCdqlEbYtN8qJDqNcLq+7C3mQjt2tiaAFBhcReuSd7x56vQ3//s2Bl10ucPshaJdH2KIU3eAc5LgtIQvUl23NXLl24BOfCNnBELYoFUEKtEM3dFEOPhkBAEiMsAUAIDHCFgAahOlgmqlw2Np+2vb3bO+yPZu1vcH2fbafzH6eULxUABhvIenv33YUgdtAZY1s3xURkxExlT2+TtL9EbFe0v3ZYwBAUTaB20CpdiNvlHRbdv82Se9P9D4AMH4I3MYpI2xD0jds77Q9nbWtjoiD2f0fS1pdwvsAALoI3EYpI2zfGRFnS7pE0jW2z88/GQvTBb3m98H2tO3Z7nFeAM2V789zc3NVlzM+CNzGKBy2EXEg+3lY0j2SNkg6ZHuNJGU/D/dYbyYipnLHeQE0VL4/dzqdqssZLwRuIxQKW9vH2D6ue1/SeyXtkXSvpC3ZYlskfa3I+2B0g14QYLlLaQGoj+h1s/X3Z72OwK2xotM1rpZ0Tzb5/FGS/iEi/tX2I5K+bPsqSc9I+kDB98GIFl93djGuvAM0g21tePD/6B0zly25zN+fdbTmp/9p4Cv9YOUUCtuIeErSWT3an5d0YZHXxsroXvSdwAXaYWLmMgK3hviExSuBC6AdJmYu47BQzRC2kETgAm1D4NYLYYtXELhAuxC49UHY4lUIXKBdCNx6IGzxGgQu0C4EbvUIW/RE4ALtQuBWi7DFkghcoF0I3OoQtlgWgQu0C4FbDcIWfRG4QLsQuCuPsB0D+bmPR71JInCBFplYZtpHlK/o3MhoAKZiBNrBtmLr16suAyPgUxgAgMQIWwAAEiNsAQBIjLAFACAxwhYAgMRGPhvZ9hmS7sw1nSbpryUdL+lPJc1l7R+LiB0jVwgAQMONHLYR8YSkSUmyvUrSAUn3SPqIpBsj4jOlVAgAQMOVtRv5Qkn7IuKZkl4PAIDWKCtsN0nannt8re3dtrfZPqGk9wAAoJEKh63toyW9T9I/Zk03STpdC7uYD0q6YYn1pm3P2p4tWgOAauX789zcXP8VgDFTxsj2EknfiYhDkhQRhyLiSETMS7pZ0oZeK0XETERMRcRUCTUAqFC+P3c6narLAWqnjLDdrNwuZNtrcs9dLmlPCe8BAEBjFboQge1jJF0kaWuu+dO2JyWFpKcXPQcAwNgpFLYR8QtJb1zU9sFCFQEA0DLMIAUAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJDYQGFre5vtw7b35NreYPs+209mP0/I2m37s7b32t5t++xUxQMA0ASDjmxvlXTxorbrJN0fEesl3Z89lqRLJK3PbtOSbipeJgAAzTVQ2EbEA5JeWNS8UdJt2f3bJL0/1357LHhY0vG215RRLAAATVTkmO3qiDiY3f+xpNXZ/bWSns0ttz9rAwBgLJVyglREhKQYZh3b07Znbc+WUQOA6uT789zcXNXlALVTJGwPdXcPZz8PZ+0HJJ2cW+6krO1VImImIqYiYqpADQBqIN+fO51O1eUAtVMkbO+VtCW7v0XS13LtH8rOSn6HpJ/mdjcDADB2jhpkIdvbJV0g6UTb+yV9XNKnJH3Z9lWSnpH0gWzxHZIulbRX0i8lfaTkmgEAaJSBwjYiNi/x1IU9lg1J1xQpCgCANhkobAFgJSz8rT4Y2wkrAcpF2AKohYjQub/zNwMv/9C+6xNWA5Sr8XMjz8/PD/UTAICV1viwnZiYGOonAAArrfEJxMgWAFB3jQ9bRrYAgLprfAIxsgUA1F3jw5aRLQCg7hqfQIxsAQB11/iwZWQLAKi7xicQI1sAQN01PmwZ2QIA6o7pGgHUgm2mYERrMdwDACAxwhYAgMQIWwAAEusbtra32T5se0+u7W9t/8D2btv32D4+az/F9q9s78puX0hZPAAATTDIyPZWSRcvartP0u9GxNsk/VDSR3PP7YuIyex2dTllAgDQXH3DNiIekPTCorZvRMTL2cOHJZ2UoDYAAFqhjGO2fyLpX3KPT7X9Xdvfsn1eCa8PAK0WEYqIqstAQoW+Z2v7ekkvS/pS1nRQ0rqIeN72OZK+avvMiHixx7rTkqaLvP8463ZOJutAHeT787p16yquplkiQrc/eK4kact5D1VcDVIZ+ZPa9ocl/aGkP47sT7KIeCkins/u75S0T9Kbe60fETMRMRURU6PWAKAe8v250+lUXQ5QOyOFre2LJf2VpPdFxC9z7R3bq7L7p0laL+mpMgoFAKCp+u5Gtr1d0gWSTrS9X9LHtXD28esl3Wdbkh7Ozjw+X9L/sP2fkuYlXR0RL/R8YQAAxkTfsI2IzT2ab1li2bsk3VW0KAAA2oSzawAASIywBQAgMcIWAIDECFsAABIjbAEASIywBQAgsULTNQIAXmvUeY6HXS+b5wANQNjWxCid0/ZQ6zGXMpBefq7jYQ27HnMpNwdhWxOj/kXLlUIAoP4I25oYdsTJVX+AerI91IiTq/6MBz6pAQBIjLAFACAxwhYAgMQIWwAAEuMEKQCNttwZ+XwPFXUxlmG7VOfk7F6gWSJCP3901ZLPH3vWEQIXtdA3WWxvs33Y9p5c2ydsH7C9K7tdmnvuo7b32n7C9h+kKnxU/f4Knp+fX8FqAIyqX9BK0s8fXcV30VELgwzjbpV0cY/2GyNiMrvtkCTbb5W0SdKZ2Tqft718b1hBg3Q6Aheov0GCtmvQ5YCU+oZtRDwg6YUBX2+jpDsi4qWI+JGkvZI2FKgPAIDGK3KA8lrbu7PdzCdkbWslPZtbZn/WBgDA2Bo1bG+SdLqkSUkHJd0w7AvYnrY9a3t2xBrGGsehUCf5/jw3N1d1OUDtjHQ2ckQc6t63fbOkr2cPD0g6ObfoSVlbr9eYkTSTvQbJMSTOmkad5Pvz1NQU/XkIw86ljGYa6RPb9prcw8sldc9UvlfSJtuvt32qpPWSvl2sRAAAmq3vyNb2dkkXSDrR9n5JH5d0ge1JSSHpaUlbJSkiHrP9ZUmPS3pZ0jURcSRN6cOLCL5zBwBYcX3DNiI292i+ZZnlPynpk0WKSmViYkLz8/PLBi4TWwD1Z1vHnnVkoK/1HDfJV/lQvbFLlYmJiVeuBdvrRtACzdAN3OUQtKiLsZyukUAF2sE2gYpGIHUAAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAILG+YWt7m+3Dtvfk2u60vSu7PW17V9Z+iu1f5Z77QsriAQBogkGuZ3urpM9Jur3bEBF/1L1v+wZJP80tvy8iJssqEACApusbthHxgO1Tej1n25I+IOnd5ZYFAEB7FD1me56kQxHxZK7tVNvftf0t2+cVfH0AABpvkN3Iy9ksaXvu8UFJ6yLiedvnSPqq7TMj4sXFK9qeljRd8P0B1EC+P69bt67iaoD6GXlka/soSVdIurPbFhEvRcTz2f2dkvZJenOv9SNiJiKmImJq1BoA1EO+P3c6narLAWqnyG7k90j6QUTs7zbY7theld0/TdJ6SU8VKxEAgGYb5Ks/2yU9JOkM2/ttX5U9tUmv3oUsSQ0x3cgAABOrSURBVOdL2p19Fegrkq6OiBfKLBgAgKYZ5GzkzUu0f7hH212S7ipeFgAA7cEMUgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIk5IqquQbbnJP1C0k+qrqUEJ4rtqJMmbMdvR0RrLgJr+2eSnqi6jhI04XdnEGzHyurZn2sRtpJke7YNF5JnO+qlLdvRJG35N2c76qXp28FuZAAAEiNsAQBIrE5hO1N1ASVhO+qlLdvRJG35N2c76qXR21GbY7YAALRVnUa2AAC0EmELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQGGELAEBihC0AAIkRtgAAJEbYAgCQWLKwtX2x7Sds77V9Xar3AQCg7hwR5b+ovUrSDyVdJGm/pEckbY6Ix0t/MwAAai7VyHaDpL0R8VRE/FrSHZI2JnovAABq7ahEr7tW0rO5x/sl/V5+AdvTkqazh+ckqgNogp9ERKfqIorI9+djjjnmnLe85S0VVwRUY+fOnT37c6qw7SsiZiTNSJLt8vdlA83xTNUFFJXvz1NTUzE7O1txRUA1bPfsz6l2Ix+QdHLu8UlZGwAAYydV2D4iab3tU20fLWmTpHsTvRcAALWWZDdyRLxs+1pJ/yZplaRtEfFYivcCAKDukh2zjYgdknaken0AAJqCGaQAAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASGzlsbZ9s+5u2H7f9mO0/y9o/YfuA7V3Z7dLyygUAoHmOKrDuy5L+MiK+Y/s4STtt35c9d2NEfKZ4eQAANN/IYRsRByUdzO7/zPb3Ja0tqzAAANqilGO2tk+R9HZJ/5E1XWt7t+1ttk9YYp1p27O2Z8uoAUB18v15bm6u6nKA2ikctraPlXSXpD+PiBcl3STpdEmTWhj53tBrvYiYiYipiJgqWgOAauX7c6fTqbocoHYKha3t12khaL8UEXdLUkQciogjETEv6WZJG4qXCQBAcxU5G9mSbpH0/Yj4u1z7mtxil0vaM3p5AAA0X5GzkX9f0gclfc/2rqztY5I2256UFJKelrS1UIUAADRckbORH5TkHk/tGL0cAADahxmkAABIjLAFACAxwhYAgMQIWwAAEiNsAQBIjLAFACAxwhYAgMQIWwAAEisygxQAYMxFRN9lFmb3HW+ELVZcv84ZEZqYYKcLUHcRoZ8/uqrvcseedWTsA5dPNKyoQf8Knp+fX4FqAIwiIgYOWkn6+aOrXllnXDGyxYoYtpPZfqVzMsoF6mOYkM3rrnPc5Hj+IU3YAkANjMuxz4hoxXYMi7BtsKYc+yyy66i7S7kO2wGkEhHaevdzfZf74hVvqjSoRh3V5v380VVjeQyXT7CGWhxg3V2u+fY6HPss4xhNHbYDSGXQoJWkrXc/V9lxzzKCtqus12kSRrYNlO9s3fv5kV83mGwzMgRqbKmgnbly7ZLPb737ucpHuBhe4bC1/bSkn0k6IunliJiy/QZJd0o6RdLTkj4QEf+v6HvhtUHbK0S7bfPz8wQuUFO9grQbsl22NXPl2tcsS+A2T1mfvu+KiMmImMoeXyfp/ohYL+n+7DEKGiRo8yYmJl5Zh12xQH0MErR5tvXFK970qrZBdz2jHlINdTZKui27f5uk9yd6n7E0zIlPjGaB+lsuaLt6BS6ao4xP4pD0Dds7bU9nbasj4mB2/8eSVi9eyfa07VnbsyXUgGWM8xfJsTLy/Xlubq7qcoDaKSNs3xkRZ0u6RNI1ts/PPxkLn/Sv+bSPiJmImMrtesYARvk6D6NbpJbvz51Op+pyGmWQUW0Xo9vmKvwpHBEHsp+HJd0jaYOkQ7bXSFL283DR9wEAoKkKha3tY2wf170v6b2S9ki6V9KWbLEtkr5W5H3QXOzCBpZHHxkPRUe2qyU9aPtRSd+W9M8R8a+SPiXpIttPSnpP9hglGOWs4io7c/6M6FHVZSYsIIVhzioeZgKMstnWsWcdKeW1xnF+5ELfs42IpySd1aP9eUkXFnlttEeRwCVoMQ6aMl9wN3CLzAA1jkErMV1jo4zyndm67KIa5fJadakdSG2QaRirHNWiOMK2QYadpGLYSTBSmpiYGGqE262XUS3aaKlJKnr9UdptG2YSjJRs67jJ+aF3KR83OT+2o1qJsG2cxYEbEa8J3cUdtuqgzevW3+9Wl3qBVJYK3HzodkO2LkGbN8wx3HEO2S4uRNBAExMTr8x7LP0mdHupY3DVrR6gKt3A7XWxgaXUIWi7uqNc9MenXkMxQgTaYZiJKuoUtBgOI9sGI0iBduhe3Wex/CEjNBuf1i3T6xgugOaJCN3+4Lm6/cFzqy4FJSBsAQBIjLAFACAxwhYAgMQIWwAAEiNsAQBIjLAFACAxvmcLABUZZq7w5fA93PojbBtmkM653PSN3ddgQgygWt3v0Q6i33JbznuojJKQEGHbMIP+hcvl6QCgPkYOW9tnSLoz13SapL+WdLykP5U0l7V/LCJ2jFwhXqXfiJQ5kYFmsL3siDQ/8mXk2nwjh21EPCFpUpJsr5J0QNI9kj4i6caI+EwpFQIA0HBlDX8ulLQvIp4p6fUAAGiNssJ2k6TtucfX2t5te5vtE3qtYHva9qzt2ZJqAFCRfH+em5vrvwIwZgqHre2jJb1P0j9mTTdJOl0Lu5gPSrqh13oRMRMRUxExVbQGANXK9+dOp1N1OUDtlDGyvUTSdyLikCRFxKGIOBIR85JulrShhPcAAKCxygjbzcrtQra9Jvfc5ZL2lPAeAAA0VqHv2do+RtJFkrbmmj9te1JSSHp60XNIjO/XAkD9FArbiPiFpDcuavtgoYpQCN+vBdqh3/dw0Sx8MgMAkBhhCwBAYoQtAACJEbYAACRG2AIAkBhhCwBAYoQtAACJEbYAACRG2AIAkBhhCwBAYoQtAACJEbYAACRG2AIAkBhhCwBAYoQtAACJEbYAACQ2UNja3mb7sO09ubY32L7P9pPZzxOydtv+rO29tnfbPjtV8QAANMGgI9tbJV28qO06SfdHxHpJ92ePJekSSeuz27Skm4qXCQBAcw0UthHxgKQXFjVvlHRbdv82Se/Ptd8eCx6WdLztNWUUCwBAExU5Zrs6Ig5m938saXV2f62kZ3PL7c/aXsX2tO1Z27MFagBQA/n+PDc3V3U5QO2UcoJURISkGHKdmYiYioipMmoAUJ18f+50OlWXA9ROkbA91N09nP08nLUfkHRybrmTsjYAAMZSkbC9V9KW7P4WSV/LtX8oOyv5HZJ+mtvdDADA2DlqkIVsb5d0gaQTbe+X9HFJn5L0ZdtXSXpG0geyxXdIulTSXkm/lPSRkmsGAKBRBgrbiNi8xFMX9lg2JF1TpCgAANqEGaQAAEiMsAUAIDHCFgCAxAhbAAASI2wBAEiMsAUAIDHCFgCAxAhbAAASI2wBAEhsoBmkAAAY1cLEgsuzvQKVVIewBQAkExE693f+pu9yD+27fgWqqQ67kVFL8/PzA/0EgCYgbFFLExMTA/0EgCbgEwu1xMgWQJsQtqglRrYA2oRPLNQSI1sAbdI3bG1vs33Y9p5c29/a/oHt3bbvsX181n6K7V/Z3pXdvpCyeLQXI1sAbTLIJ9atki5e1HafpN+NiLdJ+qGkj+ae2xcRk9nt6nLKxLhhZAugTfqGbUQ8IOmFRW3fiIiXs4cPSzopQW0YY4xsAbRJGZNa/ImkO3OPT7X9XUkvSvrvEfF/e61ke1rSdAnvD6Bi+f68bt26iqtBndhu/YQVgyg0PLB9vaSXJX0pazooaV1EvF3SX0j6B9u/1WvdiJiJiKmImCpSA4Dq5ftzp9OpuhyUaJCpFtHfyGFr+8OS/lDSH0f2vxERL0XE89n9nZL2SXpzCXWiQTieCrRDRGhi5jICtwQjha3tiyX9laT3RcQvc+0d26uy+6dJWi/pqTIKRXPYJnCBFiFwixvkqz/bJT0k6Qzb+21fJelzko6TdN+ir/icL2m37V2SviLp6oh4oecLo9UIXKBdCNxi+p4gFRGbezTfssSyd0m6q2hRaIdu4HLmMNAOEzOXaX76n1p/ObwU+BREUoxwgXZhhDsawhbJEbhAuxC4wyNssSIIXKBdCNzhELZYMQQu0C4E7uAIW6woAhdoFwJ3MIQtVhyBC7QLgdsfYYtKELhAuxC4yyNsURkCF2gXAndphC0qReAC7ULg9kbYonIELtAuBO5rEbYoXUQMfZO4WhDQJhMzl1VdQq2UcfF44FWYCxloB9uKrV+vuoxW4FMRAIDECFsAABIjbAEASIywBQAgsb5ha3ub7cO29+TaPmH7gO1d2e3S3HMftb3X9hO2/yBV4QAANMUgI9tbJV3co/3GiJjMbjskyfZbJW2SdGa2zudtryqrWAAAmqhv2EbEA5JeGPD1Nkq6IyJeiogfSdoraUOB+gAAaLwix2yvtb072818Qta2VtKzuWX2Z22vYXva9qzt2QI1AKiBfH+em5uruhygdkYN25sknS5pUtJBSTcM+wIRMRMRUxExNWINAGoi3587nU7V5VRm1NnT0H4jzSAVEYe6923fLKk7xcgBSSfnFj0pa8MShu1sEcEMTUANRYQeeee7h15vw79/M0E1qJuRPrVtr8k9vFxS90zleyVtsv1626dKWi/p28VKBACg2fqObG1vl3SBpBNt75f0cUkX2J6UFJKelrRVkiLiMdtflvS4pJclXRMRR9KUDgBAM/QN24jY3KP5lmWW/6SkTxYpCgCANuHgHwAAiRG2AAAkRtgCAJAYYQsAQGKELQAAiRG2AAAkRtgCAJAYYQsAQGIjzY2M8jAROQC0H2FbMS4qALSDbS4qgCXxSQ8AQGKELQAAiRG2AAAkRtgCAJAYYQsAQGKELQAAifUNW9vbbB+2vSfXdqftXdntadu7svZTbP8q99wXUhYPAEATDPI921slfU7S7d2GiPij7n3bN0j6aW75fRExWVaBAAA0Xd+wjYgHbJ/S6znblvQBSe8utywAANqj6DHb8yQdiognc22n2v6u7W/ZPm+pFW1P2561PVuwBgAVy/fnubm5qssBaqdo2G6WtD33+KCkdRHxdkl/IekfbP9WrxUjYiYipiJiqmANACqW78+dTqfqcoDaGTlsbR8l6QpJd3bbIuKliHg+u79T0j5Jby5aJAAATVZkZPseST+IiP3dBtsd26uy+6dJWi/pqWIlAgDQbIN89We7pIcknWF7v+2rsqc26dW7kCXpfEm7s68CfUXS1RHxQpkFAwDQNIOcjbx5ifYP92i7S9JdxcsCAKA9mEEKAIDECFsAABIjbAEASIywBQAgMcIWAIDECFsAABIjbAEASIywBQAgMcIWAIDECFsAABIjbAEASMwRUXUNsj0n6ReSflJ1LSU4UWxHnTRhO347IlpzEVjbP5P0RNV1lKAJvzuDYDtWVs/+XIuwlSTbs224kDzbUS9t2Y4macu/OdtRL03fDnYjAwCQGGELAEBidQrbmaoLKAnbUS9t2Y4macu/OdtRL43ejtocswUAoK3qNLIFAKCVKg9b2xfbfsL2XtvXVV3PMGw/bft7tnfZns3a3mD7PttPZj9PqLrOxWxvs33Y9p5cW8+6veCz2f/PbttnV1f5qy2xHZ+wfSD7P9ll+9Lccx/NtuMJ239QTdXtRn9eefTnZvTnSsPW9ipJ/1PSJZLeKmmz7bdWWdMI3hURk7lT0q+TdH9ErJd0f/a4bm6VdPGitqXqvkTS+uw2LemmFapxELfqtdshSTdm/yeTEbFDkrLfq02SzszW+Xz2+4eS0J8rc6voz7Xvz1WPbDdI2hsRT0XEryXdIWljxTUVtVHSbdn92yS9v8JaeoqIByS9sKh5qbo3Sro9Fjws6Xjba1am0uUtsR1L2Sjpjoh4KSJ+JGmvFn7/UB76cwXoz83oz1WH7VpJz+Ye78/amiIkfcP2TtvTWdvqiDiY3f+xpNXVlDa0pepu4v/Rtdkusm253X5N3I6mafq/Mf25nlrRn6sO26Z7Z0ScrYVdM9fYPj//ZCyc6t24072bWnfmJkmnS5qUdFDSDdWWgwahP9dPa/pz1WF7QNLJuccnZW2NEBEHsp+HJd2jhd0Yh7q7ZbKfh6urcChL1d2o/6OIOBQRRyJiXtLN+s2upUZtR0M1+t+Y/lw/berPVYftI5LW2z7V9tFaOOB9b8U1DcT2MbaP696X9F5Je7RQ/5ZssS2SvlZNhUNbqu57JX0oO4vxHZJ+mts9VTuLjj9droX/E2lhOzbZfr3tU7Vwgsi3V7q+lqM/1wf9uW4iotKbpEsl/VDSPknXV13PEHWfJunR7PZYt3ZJb9TC2X9PSvrfkt5Qda09at+uhV0y/6mFYx1XLVW3JGvhDNN9kr4naarq+vtsx//K6tythQ65Jrf89dl2PCHpkqrrb+ON/lxJ7fTnBvRnZpACACCxqncjAwDQeoQtAACJEbYAACRG2AIAkBhhCwBAYoQtAACJEbYAACRG2AIAkNj/Bzk+g92MywJ1AAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 576x864 with 6 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qg2FqLRGBEJT",
        "colab_type": "text"
      },
      "source": [
        "## Prepare Dataset and DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_-UTr03eAROb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import Dataset, DataLoader\n",
        "from torchvision import transforms, datasets, models\n",
        "\n",
        "class SimDataset(Dataset):\n",
        "  def __init__(self, count, transform=None):\n",
        "    self.input_images, self.target_masks = simulation.generate_random_data(192, 192, count=count)\n",
        "    self.transform = transform\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.input_images)\n",
        "\n",
        "  def __getitem__(self, idx):\n",
        "    image = self.input_images[idx]\n",
        "    mask = self.target_masks[idx]\n",
        "    if self.transform:\n",
        "      image = self.transform(image)\n",
        "\n",
        "    return [image, mask]\n",
        "\n",
        "# use the same transformations for train/val in this example\n",
        "trans = transforms.Compose([\n",
        "  transforms.ToTensor(),\n",
        "  transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225]) # imagenet\n",
        "])\n",
        "\n",
        "train_set = SimDataset(2000, transform = trans)\n",
        "val_set = SimDataset(200, transform = trans)\n",
        "\n",
        "image_datasets = {\n",
        "  'train': train_set, 'val': val_set\n",
        "}\n",
        "\n",
        "batch_size = 25\n",
        "\n",
        "dataloaders = {\n",
        "  'train': DataLoader(train_set, batch_size=batch_size, shuffle=True, num_workers=0),\n",
        "  'val': DataLoader(val_set, batch_size=batch_size, shuffle=True, num_workers=0)\n",
        "}"
      ],
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "BtkJTyxGB-XB",
        "colab_type": "text"
      },
      "source": [
        "## Check the outputs from DataLoader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CRIOwoQvBKPm",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 303
        },
        "outputId": "0ee7b026-9b8e-4fd9-d3a0-3219a6d0c55b"
      },
      "source": [
        "import torchvision.utils\n",
        "\n",
        "def reverse_transform(inp):\n",
        "  inp = inp.numpy().transpose((1, 2, 0))\n",
        "  mean = np.array([0.485, 0.456, 0.406])\n",
        "  std = np.array([0.229, 0.224, 0.225])\n",
        "  inp = std * inp + mean\n",
        "  inp = np.clip(inp, 0, 1)\n",
        "  inp = (inp * 255).astype(np.uint8)\n",
        "\n",
        "  return inp\n",
        "\n",
        "# Get a batch of training data\n",
        "inputs, masks = next(iter(dataloaders['train']))\n",
        "\n",
        "print(inputs.shape, masks.shape)\n",
        "\n",
        "plt.imshow(reverse_transform(inputs[3]))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "torch.Size([25, 3, 192, 192]) torch.Size([25, 6, 192, 192])\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7f8f538fcda0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQEAAAD8CAYAAAB3lxGOAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAZBElEQVR4nO3de3RV9Z338fc3JzknJAFCuAsooBC5dGQYBCwW5GIFnIrMuBztU4vWp+hapT7LsY5Yp8oaHxVFO6vTzqNCdaCtBbUVtX2cKlIv0ypWQFFA7uoYiigQMYRLSM53/jg7NEAiSc452Un257XWd+Wc39lnn9/PmA/7dvbP3B0Ria6csDsgIuFSCIhEnEJAJOIUAiIRpxAQiTiFgEjEZS0EzGyqmW02s21mNjdbnyMi6bFsXCdgZjFgC3AhUAa8CVzp7hsz/mEikpZsbQmMBra5+w53rwKWATOy9FkikobcLK23D/BRnedlwJiGFjYzXbYokn173L37iY3ZCoFTMrPZwOywPl8kgj6srzFbIbAT6Ffned+g7Rh3XwgsBG0JiIQpW8cE3gQGmdkAM4sDVwDPZumzRCQNWdkScPdqM5sDPA/EgEfdfUM2PktE0pOVU4RN7oR2B0Rawhp3H3Vio64YFIk4hYBIxCkERCJOISAScQoBkYhTCIhEnEJAJOIUAiIRpxAQiTiFgEjEKQREIk4hIBJxCgGRiFMIiEScQkAk4hQCIhGnEBCJOIWASMQpBEQiTiEgEnHNDgEz62dmL5nZRjPbYGb/J2ifZ2Y7zeztoKZnrrsikmnp3HK8GrjJ3deaWUdgjZmtCF77V3e/P/3uiUi2NTsE3H0XsCt4XGFm75Gag1BE2pCMHBMws/7AXwNvBE1zzOwdM3vUzLo08J7ZZrbazFZnog8i0jxpTz5iZkXAK8Bd7v6UmfUE9gAO3An0dvdvnWIdmnxEJPsyP/mImeUBvwYec/enANx9t7vXuHsSWASMTuczRCS70jk7YMAjwHvu/sM67b3rLDYTWN/87rVdiUSCgoIC4vF42F0R+ULpbAmMA64CJp1wOvA+M3vXzN4BJgI3ZqKjbc2yxx/nQGUljz32WNhdEflC6Zwd+ANg9bz0XPO7IyItTVcMikScQkAk4hQCIhGnEBCJOIWASMSl8wWiSCssLCSRSDT4ejwvL/UzHqekpKTB5Y4cOUJlZWXG+yfSaO4eepG6xLhN1SOPPupJ97Rr4aJFoY9FFZlaXd/fn3YHRCIu7S8QZaQTbfALREVFReTn5zf4+pKf/Yxp06bxm9/8hmu/1fD3pw4fPsyBAwey0UWRE2X+C0RtzYsrVzJw4ED+7113cd311zNu3Die/NWvyM/PZ9WqVeTl5bF8+XLOHT2a795wA7ffcQelpaU8//zzmBmvvfYaxcXFPPof/8GECy5gz549DVZVVRUAVUeOfOFyCgAJW6QODD76yCPs27ePFS+8wOcVFXz6yScs/eUvOXr0KA899BA1NTU89thjlJWV8fprr5HfoQOf7tnD4sWLcXcefvhhDh06xFO//jWbN20KezgiGRGpEOjYsSOxWIwOBQUcqaoiNzeXoqIizIyOnTphZhQVFZEbi5Gfn0+H/HxiOTkUdeyYen+wTGFhIXnB0X+Rti5SuwPTp0+nY8eOjBo1iiFnn03PXr2YPGUKsViMGZdcQk5ODlMuvJDu3bszbNgwRo4cSXFxMdOmTsXMuOSSS0gkEowfP57TTz897OGIZEbYpwdb8hRhnz59PDcvz7t27eqdOnf2/Px879Wrl5uZ9+3b1wHv3bu3JxIJ79y5s5eUlHheXp6fdtppDnjfvn09JyfHe/To4QUFBV/4WcufftqT7v7kk0+GfVpIpaotnSJcs3YtgwcP5kc/+hH/eOONTJw4kRdXrqRDhw5s3baNRCLBSy+/zLjzz2furbdy34IFDBs+nDdXr8bM2LxlCyUlJTy1fDl/f9llYQ9HJDPC3gpoqxcLnaq0JaBqhVXvlkDoAdCSIVC2c6cPHTrUFy9Z4j+4/XafOnWqv/32296hQwf/vKLCE4mEr1+/3idOnOh33XWXP7xwoZ8zYoR/8OGHbmb+2f793rVrV3/5lVf8f33jG1/4WYlEwgsLCz2RSIT9i1epakshMGjQII/H437aaad5t27dvLCw0AcMGOBm5qWlpW5mPnDgQC8oKPDu3bt7r+D4wFlnneWAl5aWeiwW8zPOOMM7deoU9i9UpWpq6ZjAvHnz6NWrF1//+te56KKLGDJkCDf/0z8Rj8eZf++95ObmcsvcuQwaNIiLL76Yf7j8cvr06cPtd9yBmXH3PfdQWFjInDlzGDXqpAuvRNqktK8TMLMPgAqgBqh291FmVgI8DvQHPgAud/fydD8rXevWrePQoUO8//777Nu3j/3797NhwwaSySRr16whmUyyfv16Pv/8c8p27qS8vJyDBw/yzrp1uDtvrV1LdXU1mzZtYu/evWEPRyQjMrUlMNHdR/hfrkueC6x090HAyuB56DZu3MiRI0f4748+Yvfu3VRUVLBly5Zjf/zuzpbNmzlQWcnHu3ZRVlbG4cOHee+99wBYv3491dXVbN++nfLy0DNNJDMysD//AdDthLbNpGYeAugNbG4NxwReX7XKzzrrLL//gQf8hhtu8AkTJvhzzz3n+fn5vn7DBo/H4/78Cy/4eeed59+7+Wa/++67fcjQof6HP/7Rzczfefdd79Kliy9btswvvfTSsPfvVKqmVr3HBDIxDdn7QHnwIQ+7+0Iz+8zdi4PXDSivfd7AOtLrRCMVFRVx8OBB4vE4yWSSmpoa8vPzqayspKioiAMHDlBYWMjhw4fJzc3FzKiqqqJDhw7HlqmsrKRDhw5UVVVRXV3dEt0WyZR6v0WYiS2BPsHPHsA6YDzw2QnLlNfzvtnA6qBaJAn3lZf7sOHDfdmyZX7nnXf6xRdf7Ju3bPGCggKvSSY9kUj49h07fMqUKb5gwQJfvGSJjxw50j/59FM3Mz9aXe3dunXzN954w2ddfXXYqa5SNbWysyVQl5nNAw4A3wYucPddwbRkL7t76Re8L3Od+AJ5eXkcPXqUWCx27D9ALBajurr62Gu5ubnU1NSQk5M6XJJMJsnNzeXo0aPHLZNMJkkmky3RbZFMycqEpIVm1rH2MfBVUnMPPgvMChabBTyTzudkytvr1lFaWsqPf/ITbvre95g0aRK/f+klOnTowI4dO4jH47z6X//F+eefz63f/z73P/AAw4YNY83atZgZW7dto6SkhKefflqXDUv7keauwEBSuwDrgA3AbUF7V1JnBbYCLwIlreHA4Lhx47ygoMCHDBni/fv395KSEh89erTn5OT4hAkT3Mx8zJgx3qVLFx84cKCXnn22FxUV+Ze//GUHfPz48Z6Xl+d/PXKk9+zZM+xNO5WqqZX5i4XcfYe7nxPUMHe/K2jf6+6T3X2Qu09x933pfE6mzLj0Ujp37szYsWMZPnw4vXv3ZurUqcRiMS677DJisRjTpk+nR48e/NU55zD63HPp0qULl1xyCWbG3192GfF4nMmTJ9O/f/+whyOSEZG6YjCRSGBm5MXj5ObmEovFiAdtieB+gYl4nJycHPJyc8nLy0u9FtxaPD9YNh6PE4vFwhyKSMboRqMi0aEbjYrIyRQCIhGnEBCJOIWAtHp/+OMfufTSS8PuRrulA4PS6nTr1o0XVqw49ry0tJTdu3fz2WefnbTs2DFjjk30IqdU74HBSM07IG1Dbl4eI0aMOK6tvusy3J3U99MkHdodEIk4hYBIxCkERCJOISBtWn5+vo4LpEkhIG2WmbGvvFzzQqZJISBtmpnx3qZNTJo0KeyutFkKAWnz8vPz+fkvfsG1114bdlfaJIWAtAu9e/fmpptu4h9vuinsrrQ5ulhIWp2Kzz/nlltuOfb8xhtvpFevXqd839lDhnD11VdztKqKH//4x9nsYruiEJBWp7KykgX33XfsedeuXenXr1+j39+zEYEhf6HvDohER2a/O2BmpaTmG6w1ELgdKCZ1y/FPg/bvu/tzzf0cEcmujGwJmFkM2AmMAa4BDrj7/U14v7YERLIvq7cXmwxsd/cPM7Q+EWkhmQqBK4CldZ7PMbN3zOxRM+uSoc8QkSxIOwTMLA5cAjwZND0InAmMAHYBDzTwvtlmttrMVqfbBxFpvkzMSjwD+I67f7We1/oDv3X34adYh44JiGRf1o4JXEmdXYFgAtJaM0nNTSgirVRaFwsFk5BeCFxXp/k+MxtBau6zD054TURaGV0sJBIdmoFIRE6mEBCJOIWASMTpW4SSMeeddx6LfvrTUy5XUVHBeWPHtkCPpFHcPfQidSZB1Ybra1/7mr/y6quedD9lHamq8ieefNJzcnJC73fEanW9f39hB4BCoO3XjBkz/D9/97tGBUBt1SSTfse8eV5UVBR6/yNUCgFV5mv8hAm+8ve/b1IA1K1ZV1/tJSUloY8jIqUQUGW2zjrrLH93/fpmB0BtXXnllV5cXBz6eCJQCgFVZuvTPXvSDoDauu7660MfTwSq3hDQKUKRiFMIiEScQkAk4hQCIhGnEBCJOIWASMQpBEQiTiEgEnEKAWm2mpqa2ou90pJMJjOyHmkehYA0W+9evXjrrbfSXs8FEyaw8OGHM9AjaY5GhUAwicgnZra+TluJma0ws63Bzy5Bu5nZv5nZtmACkpHZ6ryEy92ZPGkSjz/++KkXbsDwYcNYtWpVBnslTdbIa/vHAyOB9XXa7gPmBo/nAvcGj6cD/wkYMBZ4Q98daN91xhln+A9/+MMmf5X43HPP9dzc3ND7H6FK7wtEQH+OD4HNQO/gcW9gc/D4YeDK+pZTCLTfGjJ0qM+/995GBcCBykr/xlVXeXCXaVXLVb0hkM7txXq6+67g8cdAz+BxH+CjOsuVBW27kHbrvY0beeSnPyWWc+o9zMOHD/OLn/+8BXoljZGRewy6uzd17gAzmw3MzsTnS+uwdetWbr755rC7IU2UztmB3bVTjgU/PwnadwL96izXN2g7jrsvdPdRXs9kCCLSctIJgWeBWcHjWcAzddq/GZwlGAvsr7PbICKtTSMPCi4ltU9/lNQ+/rVAV2AlsBV4ESgJljXg34HtwLvAKJ0dUKlaRdV7YFBzEYpEh+YiFJGTKQREIk4hIBJxCgGRiFMIiEScQkAk4hQCIhGnEBCJOIWASMQpBEQiTiEgEnEKAZGIUwiIRJxCQCTiFAIiEacQEIk4hYBIxCkERCJOISAScQoBkYg7ZQg0MBnpAjPbFEw4utzMioP2/mZ2yMzeDuqhbHZeRNLXmC2BxcDUE9pWAMPd/a+ALcCtdV7b7u4jgro+M90UkWw5ZQi4+6vAvhPaXnD36uDpKlKzDIlIG5SJYwLfIjUVea0BZvaWmb1iZl9p6E1mNtvMVpvZ6gz0QUSaqznTktdpvw1YDscmMUkAXYPHf0NqduJOmoFI1Vbqn3/wAz946NAp66WXXw69r82ozE5NbmZXA38LTPbav2T3I8CR4PEaM9sODAb0r720aitefJEzzzyT4uJi8vPzT7n8mDFj2PH++9RUVzN48ODaf8zapuZsCZA6ULgR6H7Cct2BWPB4IKnZiEu0JaBqzfXMM894xYEDnnRvctUkk/78Cy94UVFR6ONoRDVvS8DMlgIXAN3MrAy4g9TZgASwwswAVgVnAsYD/2JmR4EkcL2776t3xSIhi8VizJ8/n4umTiUejx/32u7du1mwYMFJ75k5cybjxo079tzMuPDCC7n7nnuYf889/PnPf856vzOuMVsC2S7CT0hVxKqgoMCvueYar0kmj/uXfe1bb/niJUv87rvvrvd9V111lS9essT//3PPnbRVcNttt3n//v1DH9sXVL1bAqEHgEJA1dJV1LGjT5s27aQ/4q3btvn//va3G7WOL33pS/72unUnrePWW2/10/r0CX2MDZRCQKUCfOKkSSf98e4rL/ep06Y1aT3dunXzPXv3nrQ1MX/+/NDH2EBl9uyASFuVkzqOdUwymeS8sWPZvHlzk9azZ88eevbowaHDh4nFYgTHxzAzzKz2H7jWL+ytAG0JqFqy5nz3u360uvq4f7k7deqU1jpzcnJ87759x9ZXXVPjv/ntb0Mfaz1V75aAvkUokWJmxGKx49pqamrSWmcymTzueU5ODjk5bedPq+30VCTDqqqqGDtmDIcOHUp7XRdOmcKWLVsy0KuWp2MCElnuzp/+9KeMrGvt2rUcPHgwI+tqadoSEIk4hYBIxCkERCJOISCRduJ3BporLx4/dp1AW6MDgxJZiUSCQ4cP06ljRyorK9Na18cff0xxcXGGetaytCUgkZbJf73b6pZA6FcL6opBVUtWly5d/JuzZh13xeCmzZt9wIABzVpfTk6Ob9i48birEBcuWuR9+/YNfaz1lK4YFCkvL+ejjz46rm3w4MEsXLSI0aNHN2ldnTt35qnlyzn77LOPuwpx3969lJWVZaS/LSLsrQBtCahauvr37+/333//Sd8kXLp0qU+YMKFR6+jbt68vqGcdTzzxhE+ePDn0MTZQ+iqxSlVbXbt29aeWLz/pa8C/XLrUZ86c6ZOnTKn3feecc47PnDnTb5k796QAWPHii37++eeHPrYvKIWASlW38vLyfP369V5dU3PyDUa2bvUhQ4acVL947LF67zW48b33vLS0NPQxnaLqDYHaW4WHyszC74RE1p937aJH9+7knPDtwsZwdw4cOMDp/fqxf//+LPQuo9a4+6gTG5s7F+E8M9tZZ87B6XVeu9XMtpnZZjO7KHP9F8mO03r3ZsvWrXW3TE+pdtmamhqKO3duCwHQsEZsqo8HRnL8LcfnAd+rZ9mhwDpSdyIeAGwnuAW5dgdUrblyc3M9Ly/P77zzzkbdavy111/3vLw8z8vLC73vTajm3V7M3V81s/6nWi4wA1jmqUlI3jezbcBo4PVGvl8kFNXVqak177vvPh588MFTLl9VVcXRo0ez3a0Wkc5lw3PM7JukZhe6yd3LgT6kJiitVRa0ibQJFRUVVFRUhN2NFtXci4UeBM4ERgC7gAeaugJNSCrSOjQrBNx9t7vXuHsSWERqkx9S0471q7No36CtvnUsdPdR9R2tFJGW06wQMLPedZ7OBGrPHDwLXGFmCTMbAAwCMnP/JhHJiubORXiBmY0gdcTxA+A6AHffYGZPkJqstBr4jrundytXEckqXSwkEh3Nu1hIRNo3hYBIxCkERCJOISAScQoBkYhTCIhEnEJAJOIUAiIRpxAQiTiFgEjEKQREIk4hIBJxCgGRiFMIiEScQkAk4hQCIhGnEBCJOIWASMQpBEQirrlzET5eZx7CD8zs7aC9v5kdqvPaQ9nsvIikrzEzEC0GfgL8rLbB3f+h9rGZPQDUnY1xu7uPyFQHRSS70pqL0MwMuByYlNluiUhLSfeYwFeA3e6+tU7bADN7y8xeMbOvpLl+EcmydCYkBbgSWFrn+S7gdHffa2Z/AzxtZsPc/fMT32hms4HZaX6+iKSp2VsCZpYL/B3weG2bux9x973B4zXAdmBwfe/XXIQirUM6uwNTgE3uXlbbYGbdzSwWPB5Iai7CHel1UUSyqTGnCJcCrwOlZlZmZtcGL13B8bsCAOOBd4JThr8Crnf3fZnssIhkluYiFIkOzUUoIidTCIhEnEJAJOIUAiIRpxAQiTiFgEjEKQREIk4hIBJxCgGRiFMIiEScQkAk4hQCIhGnEBCJOIWASMSle3uxTNkDVAY/27NutO8xtvfxQdse4xn1NbaK+wkAmNnq9n6rsfY+xvY+PmifY9TugEjEKQREIq41hcDCsDvQAtr7GNv7+KAdjrHVHBMQkXC0pi0BEQlB6CFgZlPNbLOZbTOzuWH3J1OC2ZrfDWZnXh20lZjZCjPbGvzsEnY/m6KBGarrHZOl/Fvwe33HzEaG1/PGaWB888xsZ52ZtqfXee3WYHybzeyicHqdvlBDIJio5N+BacBQ4EozGxpmnzJsoruPqHNKaS6w0t0HASuD523JYmDqCW0NjWkaqclnBpGabu7BFupjOhZz8vgA/jX4PY5w9+cAgv9PrwCGBe/5f7UT77Q1YW8JjAa2ufsOd68ClgEzQu5TNs0AlgSPlwCXhtiXJnP3V4ETJ5NpaEwzgJ95yiqg2Mx6t0xPm6eB8TVkBrAsmHrvfWAbqf+f25ywQ6AP8FGd52VBW3vgwAtmtiaYfBWgp7vvCh5/DPQMp2sZ1dCY2tPvdk6wS/NonV24djO+sEOgPTvf3UeS2iz+jpmNr/uip07LtKtTM+1xTKR2Y84ERpCadfuBcLuTeWGHwE6gX53nfYO2Ns/ddwY/PwGWk9pU3F27SRz8/CS8HmZMQ2NqF79bd9/t7jXungQW8ZdN/nYxPgg/BN4EBpnZADOLkzrQ8mzIfUqbmRWaWcfax8BXgfWkxjYrWGwW8Ew4Pcyohsb0LPDN4CzBWGB/nd2GNuOE4xgzSf0eITW+K8wsYWYDSB0A/VNL9y8TQv0WobtXm9kc4HkgBjzq7hvC7FOG9ASWmxmk/hv/0t1/Z2ZvAk8EMzt/CFweYh+bLJih+gKgm5mVAXcA86l/TM8B00kdMDsIXNPiHW6iBsZ3gZmNILWb8wFwHYC7bzCzJ4CNQDXwHXevCaPf6dIVgyIRF/bugIiETCEgEnEKAZGIUwiIRJxCQCTiFAIiEacQEIk4hYBIxP0PV4yCDEHV7bQAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "E7XRZIKtCN8E",
        "colab_type": "text"
      },
      "source": [
        "# Define a UNet module"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "b8EJl0hcC5DH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import torch.nn as nn\n",
        "import torchvision.models\n",
        "\n",
        "\n",
        "def convrelu(in_channels, out_channels, kernel, padding):\n",
        "  return nn.Sequential(\n",
        "    nn.Conv2d(in_channels, out_channels, kernel, padding=padding),\n",
        "    nn.ReLU(inplace=True),\n",
        "  )\n",
        "\n",
        "\n",
        "class ResNetUNet(nn.Module):\n",
        "  def __init__(self, n_class):\n",
        "    super().__init__()\n",
        "\n",
        "    self.base_model = torchvision.models.resnet18(pretrained=True)\n",
        "    self.base_layers = list(self.base_model.children())\n",
        "\n",
        "    self.layer0 = nn.Sequential(*self.base_layers[:3]) # size=(N, 64, x.H/2, x.W/2)\n",
        "    self.layer0_1x1 = convrelu(64, 64, 1, 0)\n",
        "    self.layer1 = nn.Sequential(*self.base_layers[3:5]) # size=(N, 64, x.H/4, x.W/4)\n",
        "    self.layer1_1x1 = convrelu(64, 64, 1, 0)\n",
        "    self.layer2 = self.base_layers[5]  # size=(N, 128, x.H/8, x.W/8)\n",
        "    self.layer2_1x1 = convrelu(128, 128, 1, 0)\n",
        "    self.layer3 = self.base_layers[6]  # size=(N, 256, x.H/16, x.W/16)\n",
        "    self.layer3_1x1 = convrelu(256, 256, 1, 0)\n",
        "    self.layer4 = self.base_layers[7]  # size=(N, 512, x.H/32, x.W/32)\n",
        "    self.layer4_1x1 = convrelu(512, 512, 1, 0)\n",
        "\n",
        "    self.upsample = nn.Upsample(scale_factor=2, mode='bilinear', align_corners=True)\n",
        "\n",
        "    self.conv_up3 = convrelu(256 + 512, 512, 3, 1)\n",
        "    self.conv_up2 = convrelu(128 + 512, 256, 3, 1)\n",
        "    self.conv_up1 = convrelu(64 + 256, 256, 3, 1)\n",
        "    self.conv_up0 = convrelu(64 + 256, 128, 3, 1)\n",
        "\n",
        "    self.conv_original_size0 = convrelu(3, 64, 3, 1)\n",
        "    self.conv_original_size1 = convrelu(64, 64, 3, 1)\n",
        "    self.conv_original_size2 = convrelu(64 + 128, 64, 3, 1)\n",
        "\n",
        "    self.conv_last = nn.Conv2d(64, n_class, 1)\n",
        "\n",
        "  def forward(self, input):\n",
        "    x_original = self.conv_original_size0(input)\n",
        "    x_original = self.conv_original_size1(x_original)\n",
        "\n",
        "    layer0 = self.layer0(input)\n",
        "    layer1 = self.layer1(layer0)\n",
        "    layer2 = self.layer2(layer1)\n",
        "    layer3 = self.layer3(layer2)\n",
        "    layer4 = self.layer4(layer3)\n",
        "\n",
        "    layer4 = self.layer4_1x1(layer4)\n",
        "    x = self.upsample(layer4)\n",
        "    layer3 = self.layer3_1x1(layer3)\n",
        "    x = torch.cat([x, layer3], dim=1)\n",
        "    x = self.conv_up3(x)\n",
        "\n",
        "    x = self.upsample(x)\n",
        "    layer2 = self.layer2_1x1(layer2)\n",
        "    x = torch.cat([x, layer2], dim=1)\n",
        "    x = self.conv_up2(x)\n",
        "\n",
        "    x = self.upsample(x)\n",
        "    layer1 = self.layer1_1x1(layer1)\n",
        "    x = torch.cat([x, layer1], dim=1)\n",
        "    x = self.conv_up1(x)\n",
        "\n",
        "    x = self.upsample(x)\n",
        "    layer0 = self.layer0_1x1(layer0)\n",
        "    x = torch.cat([x, layer0], dim=1)\n",
        "    x = self.conv_up0(x)\n",
        "\n",
        "    x = self.upsample(x)\n",
        "    x = torch.cat([x, x_original], dim=1)\n",
        "    x = self.conv_original_size2(x)\n",
        "\n",
        "    out = self.conv_last(x)\n",
        "\n",
        "    return out"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "gJ65Br1oDCOX",
        "colab_type": "text"
      },
      "source": [
        "## Instantiate the UNet model\n",
        "\n",
        "- Move the model to GPU if available\n",
        "- Show model summaries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bY0Vk2VDCAiz",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 100,
          "referenced_widgets": [
            "26c23e86b3144131b1a461204794c75c",
            "da049760188f4fcbbb489bd6112c4465",
            "a56301de9c984f33afb92133a9b695ea",
            "a4a0557cdac841dabc49bb820eee9dda",
            "7f8d3d80a279490a88db38957610eb85",
            "7c346e79cfd54349b4c1741176b75d9d",
            "ddfaa576cef74773a11e1e97aa366530",
            "096d1eecb02c4ef9a6ecee295d865419"
          ]
        },
        "outputId": "263fdf19-82e4-41ed-d0f7-f2026722b234"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "import pytorch_unet\n",
        "\n",
        "device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')\n",
        "print('device', device)\n",
        "\n",
        "model = ResNetUNet(6)\n",
        "model = model.to(device)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "device cuda\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet18-5c106cde.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-5c106cde.pth\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "26c23e86b3144131b1a461204794c75c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, max=46827520.0), HTML(value='')))"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RaZdFgOnGA_p",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "0fcd0aa5-6718-4490-e595-595a78442572"
      },
      "source": [
        "model"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "ResNetUNet(\n",
              "  (base_model): ResNet(\n",
              "    (conv1): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
              "    (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (relu): ReLU(inplace=True)\n",
              "    (maxpool): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
              "    (layer1): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (layer2): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (downsample): Sequential(\n",
              "          (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "          (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        )\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (layer3): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (downsample): Sequential(\n",
              "          (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "          (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        )\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (layer4): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (downsample): Sequential(\n",
              "          (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "          (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        )\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (avgpool): AdaptiveAvgPool2d(output_size=(1, 1))\n",
              "    (fc): Linear(in_features=512, out_features=1000, bias=True)\n",
              "  )\n",
              "  (layer0): Sequential(\n",
              "    (0): Conv2d(3, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)\n",
              "    (1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    (2): ReLU(inplace=True)\n",
              "  )\n",
              "  (layer0_1x1): Sequential(\n",
              "    (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (layer1): Sequential(\n",
              "    (0): MaxPool2d(kernel_size=3, stride=2, padding=1, dilation=1, ceil_mode=False)\n",
              "    (1): Sequential(\n",
              "      (0): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "      (1): BasicBlock(\n",
              "        (conv1): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn1): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "        (relu): ReLU(inplace=True)\n",
              "        (conv2): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "        (bn2): BatchNorm2d(64, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (layer1_1x1): Sequential(\n",
              "    (0): Conv2d(64, 64, kernel_size=(1, 1), stride=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (layer2): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(64, 128, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(64, 128, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(128, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(128, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "  )\n",
              "  (layer2_1x1): Sequential(\n",
              "    (0): Conv2d(128, 128, kernel_size=(1, 1), stride=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (layer3): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(128, 256, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(128, 256, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(256, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(256, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "  )\n",
              "  (layer3_1x1): Sequential(\n",
              "    (0): Conv2d(256, 256, kernel_size=(1, 1), stride=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (layer4): Sequential(\n",
              "    (0): BasicBlock(\n",
              "      (conv1): Conv2d(256, 512, kernel_size=(3, 3), stride=(2, 2), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (downsample): Sequential(\n",
              "        (0): Conv2d(256, 512, kernel_size=(1, 1), stride=(2, 2), bias=False)\n",
              "        (1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      )\n",
              "    )\n",
              "    (1): BasicBlock(\n",
              "      (conv1): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn1): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "      (relu): ReLU(inplace=True)\n",
              "      (conv2): Conv2d(512, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1), bias=False)\n",
              "      (bn2): BatchNorm2d(512, eps=1e-05, momentum=0.1, affine=True, track_running_stats=True)\n",
              "    )\n",
              "  )\n",
              "  (layer4_1x1): Sequential(\n",
              "    (0): Conv2d(512, 512, kernel_size=(1, 1), stride=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (upsample): Upsample(scale_factor=2.0, mode=bilinear)\n",
              "  (conv_up3): Sequential(\n",
              "    (0): Conv2d(768, 512, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (conv_up2): Sequential(\n",
              "    (0): Conv2d(640, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (conv_up1): Sequential(\n",
              "    (0): Conv2d(320, 256, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (conv_up0): Sequential(\n",
              "    (0): Conv2d(320, 128, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (conv_original_size0): Sequential(\n",
              "    (0): Conv2d(3, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (conv_original_size1): Sequential(\n",
              "    (0): Conv2d(64, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (conv_original_size2): Sequential(\n",
              "    (0): Conv2d(192, 64, kernel_size=(3, 3), stride=(1, 1), padding=(1, 1))\n",
              "    (1): ReLU(inplace=True)\n",
              "  )\n",
              "  (conv_last): Conv2d(64, 6, kernel_size=(1, 1), stride=(1, 1))\n",
              ")"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MoVYhHpbCSdY",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "5ec5dba9-710a-4f0a-e9af-c3a5c35ef31a"
      },
      "source": [
        "from torchsummary import summary\n",
        "summary(model, input_size=(3, 224, 224))"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "----------------------------------------------------------------\n",
            "        Layer (type)               Output Shape         Param #\n",
            "================================================================\n",
            "            Conv2d-1         [-1, 64, 224, 224]           1,792\n",
            "              ReLU-2         [-1, 64, 224, 224]               0\n",
            "            Conv2d-3         [-1, 64, 224, 224]          36,928\n",
            "              ReLU-4         [-1, 64, 224, 224]               0\n",
            "            Conv2d-5         [-1, 64, 112, 112]           9,408\n",
            "            Conv2d-6         [-1, 64, 112, 112]           9,408\n",
            "       BatchNorm2d-7         [-1, 64, 112, 112]             128\n",
            "       BatchNorm2d-8         [-1, 64, 112, 112]             128\n",
            "              ReLU-9         [-1, 64, 112, 112]               0\n",
            "             ReLU-10         [-1, 64, 112, 112]               0\n",
            "        MaxPool2d-11           [-1, 64, 56, 56]               0\n",
            "        MaxPool2d-12           [-1, 64, 56, 56]               0\n",
            "           Conv2d-13           [-1, 64, 56, 56]          36,864\n",
            "           Conv2d-14           [-1, 64, 56, 56]          36,864\n",
            "      BatchNorm2d-15           [-1, 64, 56, 56]             128\n",
            "      BatchNorm2d-16           [-1, 64, 56, 56]             128\n",
            "             ReLU-17           [-1, 64, 56, 56]               0\n",
            "             ReLU-18           [-1, 64, 56, 56]               0\n",
            "           Conv2d-19           [-1, 64, 56, 56]          36,864\n",
            "           Conv2d-20           [-1, 64, 56, 56]          36,864\n",
            "      BatchNorm2d-21           [-1, 64, 56, 56]             128\n",
            "      BatchNorm2d-22           [-1, 64, 56, 56]             128\n",
            "             ReLU-23           [-1, 64, 56, 56]               0\n",
            "             ReLU-24           [-1, 64, 56, 56]               0\n",
            "       BasicBlock-25           [-1, 64, 56, 56]               0\n",
            "       BasicBlock-26           [-1, 64, 56, 56]               0\n",
            "           Conv2d-27           [-1, 64, 56, 56]          36,864\n",
            "           Conv2d-28           [-1, 64, 56, 56]          36,864\n",
            "      BatchNorm2d-29           [-1, 64, 56, 56]             128\n",
            "      BatchNorm2d-30           [-1, 64, 56, 56]             128\n",
            "             ReLU-31           [-1, 64, 56, 56]               0\n",
            "             ReLU-32           [-1, 64, 56, 56]               0\n",
            "           Conv2d-33           [-1, 64, 56, 56]          36,864\n",
            "           Conv2d-34           [-1, 64, 56, 56]          36,864\n",
            "      BatchNorm2d-35           [-1, 64, 56, 56]             128\n",
            "      BatchNorm2d-36           [-1, 64, 56, 56]             128\n",
            "             ReLU-37           [-1, 64, 56, 56]               0\n",
            "             ReLU-38           [-1, 64, 56, 56]               0\n",
            "       BasicBlock-39           [-1, 64, 56, 56]               0\n",
            "       BasicBlock-40           [-1, 64, 56, 56]               0\n",
            "           Conv2d-41          [-1, 128, 28, 28]          73,728\n",
            "           Conv2d-42          [-1, 128, 28, 28]          73,728\n",
            "      BatchNorm2d-43          [-1, 128, 28, 28]             256\n",
            "      BatchNorm2d-44          [-1, 128, 28, 28]             256\n",
            "             ReLU-45          [-1, 128, 28, 28]               0\n",
            "             ReLU-46          [-1, 128, 28, 28]               0\n",
            "           Conv2d-47          [-1, 128, 28, 28]         147,456\n",
            "           Conv2d-48          [-1, 128, 28, 28]         147,456\n",
            "      BatchNorm2d-49          [-1, 128, 28, 28]             256\n",
            "      BatchNorm2d-50          [-1, 128, 28, 28]             256\n",
            "           Conv2d-51          [-1, 128, 28, 28]           8,192\n",
            "           Conv2d-52          [-1, 128, 28, 28]           8,192\n",
            "      BatchNorm2d-53          [-1, 128, 28, 28]             256\n",
            "      BatchNorm2d-54          [-1, 128, 28, 28]             256\n",
            "             ReLU-55          [-1, 128, 28, 28]               0\n",
            "             ReLU-56          [-1, 128, 28, 28]               0\n",
            "       BasicBlock-57          [-1, 128, 28, 28]               0\n",
            "       BasicBlock-58          [-1, 128, 28, 28]               0\n",
            "           Conv2d-59          [-1, 128, 28, 28]         147,456\n",
            "           Conv2d-60          [-1, 128, 28, 28]         147,456\n",
            "      BatchNorm2d-61          [-1, 128, 28, 28]             256\n",
            "      BatchNorm2d-62          [-1, 128, 28, 28]             256\n",
            "             ReLU-63          [-1, 128, 28, 28]               0\n",
            "             ReLU-64          [-1, 128, 28, 28]               0\n",
            "           Conv2d-65          [-1, 128, 28, 28]         147,456\n",
            "           Conv2d-66          [-1, 128, 28, 28]         147,456\n",
            "      BatchNorm2d-67          [-1, 128, 28, 28]             256\n",
            "      BatchNorm2d-68          [-1, 128, 28, 28]             256\n",
            "             ReLU-69          [-1, 128, 28, 28]               0\n",
            "             ReLU-70          [-1, 128, 28, 28]               0\n",
            "       BasicBlock-71          [-1, 128, 28, 28]               0\n",
            "       BasicBlock-72          [-1, 128, 28, 28]               0\n",
            "           Conv2d-73          [-1, 256, 14, 14]         294,912\n",
            "           Conv2d-74          [-1, 256, 14, 14]         294,912\n",
            "      BatchNorm2d-75          [-1, 256, 14, 14]             512\n",
            "      BatchNorm2d-76          [-1, 256, 14, 14]             512\n",
            "             ReLU-77          [-1, 256, 14, 14]               0\n",
            "             ReLU-78          [-1, 256, 14, 14]               0\n",
            "           Conv2d-79          [-1, 256, 14, 14]         589,824\n",
            "           Conv2d-80          [-1, 256, 14, 14]         589,824\n",
            "      BatchNorm2d-81          [-1, 256, 14, 14]             512\n",
            "      BatchNorm2d-82          [-1, 256, 14, 14]             512\n",
            "           Conv2d-83          [-1, 256, 14, 14]          32,768\n",
            "           Conv2d-84          [-1, 256, 14, 14]          32,768\n",
            "      BatchNorm2d-85          [-1, 256, 14, 14]             512\n",
            "      BatchNorm2d-86          [-1, 256, 14, 14]             512\n",
            "             ReLU-87          [-1, 256, 14, 14]               0\n",
            "             ReLU-88          [-1, 256, 14, 14]               0\n",
            "       BasicBlock-89          [-1, 256, 14, 14]               0\n",
            "       BasicBlock-90          [-1, 256, 14, 14]               0\n",
            "           Conv2d-91          [-1, 256, 14, 14]         589,824\n",
            "           Conv2d-92          [-1, 256, 14, 14]         589,824\n",
            "      BatchNorm2d-93          [-1, 256, 14, 14]             512\n",
            "      BatchNorm2d-94          [-1, 256, 14, 14]             512\n",
            "             ReLU-95          [-1, 256, 14, 14]               0\n",
            "             ReLU-96          [-1, 256, 14, 14]               0\n",
            "           Conv2d-97          [-1, 256, 14, 14]         589,824\n",
            "           Conv2d-98          [-1, 256, 14, 14]         589,824\n",
            "      BatchNorm2d-99          [-1, 256, 14, 14]             512\n",
            "     BatchNorm2d-100          [-1, 256, 14, 14]             512\n",
            "            ReLU-101          [-1, 256, 14, 14]               0\n",
            "            ReLU-102          [-1, 256, 14, 14]               0\n",
            "      BasicBlock-103          [-1, 256, 14, 14]               0\n",
            "      BasicBlock-104          [-1, 256, 14, 14]               0\n",
            "          Conv2d-105            [-1, 512, 7, 7]       1,179,648\n",
            "          Conv2d-106            [-1, 512, 7, 7]       1,179,648\n",
            "     BatchNorm2d-107            [-1, 512, 7, 7]           1,024\n",
            "     BatchNorm2d-108            [-1, 512, 7, 7]           1,024\n",
            "            ReLU-109            [-1, 512, 7, 7]               0\n",
            "            ReLU-110            [-1, 512, 7, 7]               0\n",
            "          Conv2d-111            [-1, 512, 7, 7]       2,359,296\n",
            "          Conv2d-112            [-1, 512, 7, 7]       2,359,296\n",
            "     BatchNorm2d-113            [-1, 512, 7, 7]           1,024\n",
            "     BatchNorm2d-114            [-1, 512, 7, 7]           1,024\n",
            "          Conv2d-115            [-1, 512, 7, 7]         131,072\n",
            "          Conv2d-116            [-1, 512, 7, 7]         131,072\n",
            "     BatchNorm2d-117            [-1, 512, 7, 7]           1,024\n",
            "     BatchNorm2d-118            [-1, 512, 7, 7]           1,024\n",
            "            ReLU-119            [-1, 512, 7, 7]               0\n",
            "            ReLU-120            [-1, 512, 7, 7]               0\n",
            "      BasicBlock-121            [-1, 512, 7, 7]               0\n",
            "      BasicBlock-122            [-1, 512, 7, 7]               0\n",
            "          Conv2d-123            [-1, 512, 7, 7]       2,359,296\n",
            "          Conv2d-124            [-1, 512, 7, 7]       2,359,296\n",
            "     BatchNorm2d-125            [-1, 512, 7, 7]           1,024\n",
            "     BatchNorm2d-126            [-1, 512, 7, 7]           1,024\n",
            "            ReLU-127            [-1, 512, 7, 7]               0\n",
            "            ReLU-128            [-1, 512, 7, 7]               0\n",
            "          Conv2d-129            [-1, 512, 7, 7]       2,359,296\n",
            "          Conv2d-130            [-1, 512, 7, 7]       2,359,296\n",
            "     BatchNorm2d-131            [-1, 512, 7, 7]           1,024\n",
            "     BatchNorm2d-132            [-1, 512, 7, 7]           1,024\n",
            "            ReLU-133            [-1, 512, 7, 7]               0\n",
            "            ReLU-134            [-1, 512, 7, 7]               0\n",
            "      BasicBlock-135            [-1, 512, 7, 7]               0\n",
            "      BasicBlock-136            [-1, 512, 7, 7]               0\n",
            "          Conv2d-137            [-1, 512, 7, 7]         262,656\n",
            "            ReLU-138            [-1, 512, 7, 7]               0\n",
            "        Upsample-139          [-1, 512, 14, 14]               0\n",
            "          Conv2d-140          [-1, 256, 14, 14]          65,792\n",
            "            ReLU-141          [-1, 256, 14, 14]               0\n",
            "          Conv2d-142          [-1, 512, 14, 14]       3,539,456\n",
            "            ReLU-143          [-1, 512, 14, 14]               0\n",
            "        Upsample-144          [-1, 512, 28, 28]               0\n",
            "          Conv2d-145          [-1, 128, 28, 28]          16,512\n",
            "            ReLU-146          [-1, 128, 28, 28]               0\n",
            "          Conv2d-147          [-1, 256, 28, 28]       1,474,816\n",
            "            ReLU-148          [-1, 256, 28, 28]               0\n",
            "        Upsample-149          [-1, 256, 56, 56]               0\n",
            "          Conv2d-150           [-1, 64, 56, 56]           4,160\n",
            "            ReLU-151           [-1, 64, 56, 56]               0\n",
            "          Conv2d-152          [-1, 256, 56, 56]         737,536\n",
            "            ReLU-153          [-1, 256, 56, 56]               0\n",
            "        Upsample-154        [-1, 256, 112, 112]               0\n",
            "          Conv2d-155         [-1, 64, 112, 112]           4,160\n",
            "            ReLU-156         [-1, 64, 112, 112]               0\n",
            "          Conv2d-157        [-1, 128, 112, 112]         368,768\n",
            "            ReLU-158        [-1, 128, 112, 112]               0\n",
            "        Upsample-159        [-1, 128, 224, 224]               0\n",
            "          Conv2d-160         [-1, 64, 224, 224]         110,656\n",
            "            ReLU-161         [-1, 64, 224, 224]               0\n",
            "          Conv2d-162          [-1, 6, 224, 224]             390\n",
            "================================================================\n",
            "Total params: 28,976,646\n",
            "Trainable params: 28,976,646\n",
            "Non-trainable params: 0\n",
            "----------------------------------------------------------------\n",
            "Input size (MB): 0.57\n",
            "Forward/backward pass size (MB): 417.65\n",
            "Params size (MB): 110.54\n",
            "Estimated Total Size (MB): 528.76\n",
            "----------------------------------------------------------------\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "H7rAEQCUEI2v",
        "colab_type": "text"
      },
      "source": [
        "# Define the main training loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tjt9JeTuDY6D",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from collections import defaultdict\n",
        "import torch.nn.functional as F\n",
        "from loss import dice_loss\n",
        "\n",
        "checkpoint_path = \"checkpoint.pth\"\n",
        "\n",
        "def calc_loss(pred, target, metrics, bce_weight=0.5):\n",
        "    bce = F.binary_cross_entropy_with_logits(pred, target)\n",
        "\n",
        "    pred = torch.sigmoid(pred)\n",
        "    dice = dice_loss(pred, target)\n",
        "\n",
        "    loss = bce * bce_weight + dice * (1 - bce_weight)\n",
        "\n",
        "    metrics['bce'] += bce.data.cpu().numpy() * target.size(0)\n",
        "    metrics['dice'] += dice.data.cpu().numpy() * target.size(0)\n",
        "    metrics['loss'] += loss.data.cpu().numpy() * target.size(0)\n",
        "\n",
        "    return loss\n",
        "\n",
        "def print_metrics(metrics, epoch_samples, phase):\n",
        "    outputs = []\n",
        "    for k in metrics.keys():\n",
        "        outputs.append(\"{}: {:4f}\".format(k, metrics[k] / epoch_samples))\n",
        "\n",
        "    print(\"{}: {}\".format(phase, \", \".join(outputs)))\n",
        "\n",
        "def train_model(model, optimizer, scheduler, num_epochs=25):\n",
        "    best_loss = 1e10\n",
        "\n",
        "    for epoch in range(num_epochs):\n",
        "        print('Epoch {}/{}'.format(epoch, num_epochs - 1))\n",
        "        print('-' * 10)\n",
        "\n",
        "        since = time.time()\n",
        "\n",
        "        # Each epoch has a training and validation phase\n",
        "        for phase in ['train', 'val']:\n",
        "            if phase == 'train':\n",
        "                model.train()  # Set model to training mode\n",
        "            else:\n",
        "                model.eval()   # Set model to evaluate mode\n",
        "\n",
        "            metrics = defaultdict(float)\n",
        "            epoch_samples = 0\n",
        "\n",
        "            for inputs, labels in dataloaders[phase]:\n",
        "                inputs = inputs.to(device)\n",
        "                labels = labels.to(device)\n",
        "\n",
        "                # zero the parameter gradients\n",
        "                optimizer.zero_grad()\n",
        "\n",
        "                # forward\n",
        "                # track history if only in train\n",
        "                with torch.set_grad_enabled(phase == 'train'):\n",
        "                    outputs = model(inputs)\n",
        "                    loss = calc_loss(outputs, labels, metrics)\n",
        "\n",
        "                    # backward + optimize only if in training phase\n",
        "                    if phase == 'train':\n",
        "                        loss.backward()\n",
        "                        optimizer.step()\n",
        "\n",
        "                # statistics\n",
        "                epoch_samples += inputs.size(0)\n",
        "\n",
        "            print_metrics(metrics, epoch_samples, phase)\n",
        "            epoch_loss = metrics['loss'] / epoch_samples\n",
        "\n",
        "            if phase == 'train':\n",
        "              scheduler.step()\n",
        "              for param_group in optimizer.param_groups:\n",
        "                  print(\"LR\", param_group['lr'])\n",
        "\n",
        "            # save the model weights\n",
        "            if phase == 'val' and epoch_loss < best_loss:\n",
        "                print(f\"saving best model to {checkpoint_path}\")\n",
        "                best_loss = epoch_loss\n",
        "                torch.save(model.state_dict(), checkpoint_path)\n",
        "\n",
        "        time_elapsed = time.time() - since\n",
        "        print('{:.0f}m {:.0f}s'.format(time_elapsed // 60, time_elapsed % 60))\n",
        "\n",
        "    print('Best val loss: {:4f}'.format(best_loss))\n",
        "\n",
        "    # load best model weights\n",
        "    model.load_state_dict(torch.load(checkpoint_path))\n",
        "    return model"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "adcdAu9ZEOLG",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RfxgL303EMiy",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "513c2eba-d6b6-4b5f-d223-590b2c074ac6"
      },
      "source": [
        "import torch\n",
        "import torch.optim as optim\n",
        "from torch.optim import lr_scheduler\n",
        "import time\n",
        "\n",
        "num_class = 6\n",
        "model = ResNetUNet(num_class).to(device)\n",
        "\n",
        "# freeze backbone layers\n",
        "for l in model.base_layers:\n",
        "  for param in l.parameters():\n",
        "    param.requires_grad = False\n",
        "\n",
        "optimizer_ft = optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=1e-4)\n",
        "\n",
        "exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=8, gamma=0.1)\n",
        "\n",
        "model = train_model(model, optimizer_ft, exp_lr_scheduler, num_epochs=10)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch 0/9\n",
            "----------\n",
            "train: bce: 0.105508, dice: 0.960730, loss: 0.533119\n",
            "LR 0.0001\n",
            "val: bce: 0.021507, dice: 0.793499, loss: 0.407503\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 1/9\n",
            "----------\n",
            "train: bce: 0.013632, dice: 0.554114, loss: 0.283873\n",
            "LR 0.0001\n",
            "val: bce: 0.004518, dice: 0.269733, loss: 0.137125\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 2/9\n",
            "----------\n",
            "train: bce: 0.003263, dice: 0.147794, loss: 0.075529\n",
            "LR 0.0001\n",
            "val: bce: 0.002220, dice: 0.087164, loss: 0.044692\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 3/9\n",
            "----------\n",
            "train: bce: 0.002458, dice: 0.081096, loss: 0.041777\n",
            "LR 0.0001\n",
            "val: bce: 0.001856, dice: 0.059757, loss: 0.030807\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 4/9\n",
            "----------\n",
            "train: bce: 0.002173, dice: 0.062749, loss: 0.032461\n",
            "LR 0.0001\n",
            "val: bce: 0.001647, dice: 0.048988, loss: 0.025317\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 5/9\n",
            "----------\n",
            "train: bce: 0.001924, dice: 0.052670, loss: 0.027297\n",
            "LR 0.0001\n",
            "val: bce: 0.001413, dice: 0.040671, loss: 0.021042\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 6/9\n",
            "----------\n",
            "train: bce: 0.001629, dice: 0.044960, loss: 0.023295\n",
            "LR 0.0001\n",
            "val: bce: 0.001336, dice: 0.037718, loss: 0.019527\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 7/9\n",
            "----------\n",
            "train: bce: 0.001435, dice: 0.040354, loss: 0.020895\n",
            "LR 1e-05\n",
            "val: bce: 0.001220, dice: 0.035549, loss: 0.018385\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 8/9\n",
            "----------\n",
            "train: bce: 0.001252, dice: 0.036088, loss: 0.018670\n",
            "LR 1e-05\n",
            "val: bce: 0.001170, dice: 0.033778, loss: 0.017474\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Epoch 9/9\n",
            "----------\n",
            "train: bce: 0.001239, dice: 0.035437, loss: 0.018338\n",
            "LR 1e-05\n",
            "val: bce: 0.001162, dice: 0.033512, loss: 0.017337\n",
            "saving best model to checkpoint.pth\n",
            "0m 22s\n",
            "Best val loss: 0.017337\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lcRgjfk5D-kP",
        "colab_type": "text"
      },
      "source": [
        "## Predict new images using the trained model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xXRtpxHRET-v",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 68
        },
        "outputId": "3e1520e5-fec2-401d-dde2-8900b760f442"
      },
      "source": [
        "import math\n",
        "\n",
        "model.eval()   # Set model to the evaluation mode\n",
        "\n",
        "# Create a new simulation dataset for testing\n",
        "test_dataset = SimDataset(3, transform = trans)\n",
        "test_loader = DataLoader(test_dataset, batch_size=3, shuffle=False, num_workers=0)\n",
        "\n",
        "# Get the first batch\n",
        "inputs, labels = next(iter(test_loader))\n",
        "inputs = inputs.to(device)\n",
        "labels = labels.to(device)\n",
        "print('inputs.shape', inputs.shape)\n",
        "print('labels.shape', labels.shape)\n",
        "\n",
        "# Predict\n",
        "pred = model(inputs)\n",
        "# The loss functions include the sigmoid function.\n",
        "pred = torch.sigmoid(pred)\n",
        "pred = pred.data.cpu().numpy()\n",
        "print('pred.shape', pred.shape)\n",
        "\n",
        "# Change channel-order and make 3 channels for matplot\n",
        "input_images_rgb = [reverse_transform(x) for x in inputs.cpu()]\n",
        "\n",
        "# Map each channel (i.e. class) to each color\n",
        "target_masks_rgb = [helper.masks_to_colorimg(x) for x in labels.cpu().numpy()]\n",
        "pred_rgb = [helper.masks_to_colorimg(x) for x in pred]"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "inputs.shape torch.Size([3, 3, 192, 192])\n",
            "labels.shape torch.Size([3, 6, 192, 192])\n",
            "pred.shape (3, 6, 192, 192)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XPQyJc4YD39T",
        "colab_type": "text"
      },
      "source": [
        "## Left: Input image, Middle: Correct mask (Ground-truth), Rigth: Predicted mask"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "z6dkJZLBCv4t",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 704
        },
        "outputId": "7029cb1c-8210-4278-ca11-c6fa92954056"
      },
      "source": [
        "helper.plot_side_by_side([input_images_rgb, target_masks_rgb, pred_rgb])"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAsEAAAKvCAYAAACLTxJeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdf6z8d10n+ufr+636B7AB5GxTC98tYNWId/m6nNv1suJFEEUiIpCwbTZalOy35EKy/kj2ItwsZm80xpXFvTGLfgkNkGgtuy3Kmu4uDdfIRWHlW6y1INWWLaGltl8pCv4Iu/2e1/3jO6dMT885M+fMzJmZ83k8ksmZec9nZl7zTV99P+c9n/l8qrsDAABDcmLZBQAAwFETggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABichYXgqnppVd1VVXdX1ZsW9ToAAHBQtYjjBFfVySR/muQlSe5L8vEk13T3p+b+YgAAcECLWgm+Ksnd3f2Z7v4fSX4jySsW9FoAAHAglyzoeS9P8rmx2/cl+cd7bVxVTlvHkP1Fd28su4iDeNrTntZXXHHFssuApbjtttvWqmf1K0O2X78uKgRPVFVnkpxZ1uvDCvnssguYxnjPnjp1KufOnVtyRbAcVbXyPatf4aL9+nVRu0Pcn+QZY7efPhp7VHef7e7N7t5cUA3AHI337MbG2iyCwSDpV5hsUSH440murKpnVtXXJrk6yQcW9FoAAHAgC9kdorsfqao3JvmvSU4mub67P7mI1wIAgINa2D7B3X1LklsW9fwAAHBYzhgHAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQCspO5edgnAMXboEFxVz6iq36mqT1XVJ6vqX4zGf6aq7q+q20eXl82vXDi8C1tbyy4BmFJ358TZlwvCsCa6e+36dZaV4EeS/FR3f2uS70jyhqr61tF9b+/u06PLLTNXCXNQVYIwrBlBGFbf9ofWdevXQ4fg7n6guz8xuv7lJH+S5PJ5FQaLIAjD+lm3iRWGbJ36dS77BFfVFUm+Pcl/Gw29saruqKrrq+op83gNmBdBGNbPOk2sMHTr0q8zh+CqemKSm5L8eHd/Kck7kjw7yekkDyR52x6PO1NV56rq3Kw1wEEJwgc33rPnz59fdjkM0LpMrKtAv7Js69CvM4XgqvqaXAzAv9bdNydJdz/Y3Re6eyvJO5Nctdtju/tsd2929+YsNcBhCcIHM96zGxsbyy6HgVqHiXUV6FdWwar36yxHh6gk70ryJ939b8fGLxvb7JVJ7jx8ebBYgjCsn1WfWIGvWuV+nWUl+J8k+eEkL9pxOLRfqKo/rqo7knx3kp+YR6GwKIIwrJ9VnliBx1rVfr3ksA/s7o8kqV3uckg01s52ED55wvljYF2cOPvybJ35T7n4xSSwylaxX834MGJFGNbPqq4wAY+3av0qBMMYQRjWz6pNrMDeVqlfhWDYQRCG9bNKEyuwv1XpVyEYdiEIw/pZlYkVmGwV+lUIhj0IwrB+VmFiBaaz7H4VgmEfgjCsn2VPrMD0ltmvQjBMIAjD+hGEYX0sq1+FYJiCIAzrRxCG9bGMfhWCYUqCMKwfQRjWx1H3qxAMByAIw/oRhGF9HGW/CsFwQIIwrB9BGNbHibMvP5rXOZJXgRXQ3XO7JBGEYc0c1cQKrIdLll0AHJWTJ3zmg3VRVenrfnvZZQBTWNd+lQoAABgcIRgAgMERggEAGJyZ9wmuqnuTfDnJhSSPdPdmVT01yY1Jrkhyb5LXdPcXZ30tAACYh3mtBH93d5/u7s3R7Tcl+VB3X5nkQ6PbAACwEha1O8QrkrxndP09SX5oQa8DAAAHNo8Q3Ek+WFW3VdWZ0dil3f3A6PqfJ7l0Dq8DAABzMY/jBH9nd99fVX8/ya1V9enxO7u7q+pxp+kZBeYzO8eB1TTes6dOnTrS1z7Mmb6qagGVwHrQrzDZzCG4u+8f/X2oqt6f5KokD1bVZd39QFVdluShXR53NsnZJNktJC/S1gEbtLudaIHBG+/Zzc3NI+vZ7s7Hv/NFB37cVb/3OwuoBtaDfoXJZkp2VfWEqnrS9vUk35vkziQfSHLtaLNrk/zWLK8DAADzNOtK8KVJ3j/6GuOSJL/e3f+lqj6e5H1V9bokn03ymhlfBwAA5mamENzdn0ny3F3Gv5DkxbM8NwAALIodXQEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABiceZwxDgDgWDjMGe+2OfPdehGCARZolgl1m4kVjkZ35/nf+HOHfvzv3/1m/bpGhOA1c2FrKydPnJj5L7B4s06o2z56z1vmUA2wn3n06/O/8ecE4TUyyDTU3Qe6rJLtADvrXwDgonl9YE0yt+dh8Qa5ErzOQdBKMENTVbnq935n2WUAU9CvrBNpaM1YCQYAmJ1EtGYubG3N5S8AwJAJwWvGSjAAwOwkojVjJRgAYHZC8JqxEgwAMDuJaM1YCQYAmN2hD5FWVd+c5MaxoWcl+VdJnpzknyc5Pxp/c3ffcugKeQwrwQAAszt0Iuruu7r7dHefTvK8JH+b5P2ju9++fZ8APF9WggEAZjevZcEXJ7mnuz87p+djD1aCAQBmN69EdHWSG8Zuv7Gq7qiq66vqKXN6DWIlGABgHmYOwVX1tUl+MMl/GA29I8mzk5xO8kCSt+3xuDNVda6qzs1aw5BYCWZZxnv2/Pnzkx8ALI1+hcnmkYi+P8knuvvBJOnuB7v7QndvJXlnkqt2e1B3n+3uze7enEMNg2ElmGUZ79mNjY1llwPsQ7/CZPMIwddkbFeIqrps7L5XJrlzDq/BiJVgAFhdH73nLcsugSnNlIiq6glJXpLk5rHhX6iqP66qO5J8d5KfmOU1eCwrwQAwX1WV37/7zcsugyN26OMEJ0l3/02Sr98x9sMzVcS+rATD+qgqq0KwJraD8PO/8ecO9Xi9vn5mCsEAAMeFD67DYlkQAIDBEYIBABgcIRgAgMGxTzAwON09cZuqOoJKgEn0K4siBM/B1oQG7W5HZYAVsD2ZXnfz/fnep7x9j22SW//yJ/Krr7o8ickVluWr/fr5idv+6qu+IYl+5WCE4BlMCr/bqipb3cIwLMljJ9PeMwAnSVXykie/Pdfd/BNJyuQKR2zv8Lv3nHvdzfdHv3JQQvAhTBt+dxKG4eh199hkulsAHu/nixPndhC+9S9/8tHH/uqrvsHECgv22H59zD2j3u1831N/abRt8sEv/niSevQbHP3KQUhiB3TYADyuqpy5DY7AfhPq9uX7nvpL+b6n/lK+9ym/9Jjxna67+fNT7ZsIHM5BAnBy8cPqxb796gfXbfqVaVgJPoB5BOBt20HYijAsxn4T6vhEuq0qO1aYHn/G9+tu/nzOvvryeZcKg7d3v27ff7FHt/9OQ78yiQQ2g4N+yvSpFJZvtwC808UVpr1+OKeP4WhVbv3Li/vof/CLP57uPHrZ/rB6cXeIn3zcI/Ur+xGCp7RzFfig+/Vubz/ekHaLgMXYa1Vp+4gP06j66i/Ox03zS3VgepNWgb+6XXIxCP/Eo5ft8d0CcKJf2Z8QfAiz/LBtZxAGjsZhvhatql2DsB6GxTr76st39N7F1eDxVeDty3YAfvxjLtKv7EUInsL4KvDOAHyY5hp/vNVgmK/dVpVm2S9wtyBsdQnm42D9Wrn1L3/ycZfHbKFfOQAheEbTrOw6JBqshqrKE597Yaptn3Tah1NYpr2+idmNH8BxGJLZHGwH4b0uuwVgX8/AckwThAVgWA3TBGEBmMOa6hBpVXV9kh9I8lB3f9to7KlJbkxyRZJ7k7ymu79YF49O/e+SvCzJ3yZ5bXd/Yv6lH739VnQPutJ78sSJuR5yDdjdbhNkVR0o6G5PxL5WhcXaq18PEnT1K9OaNrm9O8lLd4y9KcmHuvvKJB8a3U6S709y5ehyJsk7Zi8TAADmZ6oQ3N0fTvLwjuFXJHnP6Pp7kvzQ2Ph7+6KPJXlyVV02j2IBDsquR7A+9CtHaZZ9gi/t7gdG1/88yaWj65cn+dzYdveNxtbePI/kYFcIOBrz+Ep02uOYArPRrxylufwwri9+dDtQqquqM1V1rqrOzaMGYLHGe/b8+fPLLgfYh36FyWYJwQ9u7+Yw+vvQaPz+JM8Y2+7po7HH6O6z3b3Z3Zsz1HAkfD0Dj+3ZjY2NZZdzIHqYodGvMNksIfgDSa4dXb82yW+Njf9IXfQdSf5qbLeJtTTvk1vsd/INYDZ7HSz/sBPrvE++AXyVfmWZpkpfVXVDko8m+eaquq+qXpfk55O8pKr+LMn3jG4nyS1JPpPk7iTvTPJ/zL3qJRhvyFmCsH2BYTkOM7HatxCWQ79yFKY9OsQ13X1Zd39Ndz+9u9/V3V/o7hd395Xd/T3d/fBo2+7uN3T3s7v7f+nuY7HP787V2sME4Z0B2CowLMZeB9g/yMS614RqVQnmS7+yLBLYAexsxoMEYSvAsBq2J9bty7jxcStKsHz6lUWa6oxxXHTyxIlc2NrKxZPiXVRVjwbcnSu7+wVfq8CwWPudNWp8bHwFatJEalUJFkO/sgxC8AFtB+EkjwnD27cnrfhuf5IVgGHxpjl96rQrSCZUWCz9ylETgg9hO8DuFYZ3I/zCclRVzr768kN/ZWoyhaOjXzlKQvAMdobhabYFluOgk6vJFJZHv3IUhOA5EHBhfWxPrsDq068skvQGAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADM7EEFxV11fVQ1V159jYv6mqT1fVHVX1/qp68mj8iqr6u6q6fXT5lUUWDwAAhzHNSvC7k7x0x9itSb6tu/9hkj9N8tNj993T3adHl9fPp0wAAJifiSG4uz+c5OEdYx/s7kdGNz+W5OkLqA0AABZiHvsE/1iS/zx2+5lV9YdV9btV9YI5PD8AAMzVJbM8uKrekuSRJL82Gnogyanu/kJVPS/Jb1bVc7r7S7s89kySM7O8PnB0xnv21KlTS64G2I9+hckOvRJcVa9N8gNJ/ll3d5J091e6+wuj67cluSfJN+32+O4+292b3b152BqAozPesxsbG8suB9iHfoXJDhWCq+qlSf5lkh/s7r8dG9+oqpOj689KcmWSz8yjUAAAmJeJu0NU1Q1JXpjkaVV1X5K35uLRIL4uya1VlSQfGx0J4ruS/Ouq+p9JtpK8vrsf3vWJAQBgSSaG4O6+Zpfhd+2x7U1Jbpq1KAAAWCRnjAMAYHCEYAAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHCEYAAABkcIBgBgcIRgAB6nu9Pdyy4DmIJ+PRwhmCOx1Z0LW1vLLgOYQnfnvR95ft77kecvuxRgAv16eEIwAACDIwQDADA4QjAAAIMjBAMAMDhCMAAAgzMxBFfV9VX1UFXdOTb2M1V1f1XdPrq8bOy+n66qu6vqrqr6vkUVDgAAhzXNSvC7k7x0l/G3d/fp0eWWJKmqb01ydZLnjB7z76vq5LyKBQCAebhk0gbd/eGqumLK53tFkt/o7q8k+e9VdXeSq5J89NAVAjBXBzmo/jTbVtUs5QD70K+LMzEE7+ONVfUjSc4l+anu/mKSy5N8bGyb+0ZjHGNbUzZoVU3ctrtz8oRd1WFRtg+sP61ptr32BdY5YBH062IdNm28I8mzk5xO8kCStx30CarqTFWdq6pzh6yBFbF9usb9LtNuy+oa79nz588vuxxgH/oVJjvUSnB3P7h9varemeS3RzfvT/KMsU2fPhrb7TnOJjk7eg7pZ41Ns3K7NQq5VnnX13jPbm5u6tk1VVUTV4LGV5+sGq0n/Xo86NfFOlQiqarLxm6+Msn2kSM+kOTqqvq6qnpmkiuT/MFsJQIAwHxNXAmuqhuSvDDJ06rqviRvTfLCqjqdpJPcm+S6JOnuT1bV+5J8KskjSd7Q3RcWUzoAABzONEeHuGaX4Xfts/3PJvnZWYoCAIBFsoMmAACDIwQDADA4QjAAAIMjBAMAMDhCMEfCiTAAgFUyy2mTYWpOkgHrY5oD9AOrQb8enmQCAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADM7EEFxV11fVQ1V159jYjVV1++hyb1XdPhq/oqr+buy+X1lk8QAAcBiXTLHNu5P8cpL3bg909z/dvl5Vb0vyV2Pb39Pdp+dVIAAAzNvEENzdH66qK3a7r6oqyWuSvGi+ZQEAwOLMuk/wC5I82N1/Njb2zKr6w6r63ap6wYzPDwAAczfN7hD7uSbJDWO3H0hyqru/UFXPS/KbVfWc7v7SzgdW1ZkkZ2Z8feCIjPfsqVOnllwNsB/9CpMdeiW4qi5J8qokN26PdfdXuvsLo+u3JbknyTft9vjuPtvdm929edgagKMz3rMbGxvLLgfYh36FyWbZHeJ7kny6u+/bHqiqjao6Obr+rCRXJvnMbCUCAMB8TXOItBuSfDTJN1fVfVX1utFdV+exu0IkyXcluWN0yLT/mOT13f3wPAsGAIBZTXN0iGv2GH/tLmM3Jblp9rIAAGBxnDEOAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwanuXnYNqarzSf4myV8su5Y5eFq8j1WyDu/jH3T3xrKLOIiq+nKSu5Zdxxysw38f0/A+jtZa9ax+XTnex9Has18vOepKdtPdG1V1rrs3l13LrLyP1XJc3scKuus4/Lsel/8+vA8m0K8rxPtYHXaHAABgcIRgAAAGZ5VC8NllFzAn3sdqOS7vY9Ucl39X72O1HJf3sWqOy7+r97Fa1v59rMQP4wAA4Cit0kowAAAcCSEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGJyFheCqemlV3VVVd1fVmxb1OgAAcFDV3fN/0qqTSf40yUuS3Jfk40mu6e5Pzf3FAADggBa1EnxVkru7+zPd/T+S/EaSVyzotQAA4EAWFYIvT/K5sdv3jcYAAGDpLlnWC1fVmSRnRjeft6w6YAX8RXdvLLuIScZ79glPeMLzvuVbvmXJFcFy3HbbbSvfs/oVLtqvXxcVgu9P8oyx208fjT2qu88mOZskVTX/HZNhfXx22QVMY7xnNzc3+9y5c0uuCJajqla+Z/UrXLRfvy5qd4iPJ7myqp5ZVV+b5OokH1jQawEAwIEsZCW4ux+pqjcm+a9JTia5vrs/uYjXAgCAg1rYPsHdfUuSWxb1/AAAcFjOGAcAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAcK92d7l52GcAUltmvQjBMsNWdC1tbyy4DmEJ3570feX7e+5HnL7sUYIJl96sQDADA4Bw6BFfVM6rqd6rqU1X1yar6F6Pxn6mq+6vq9tHlZfMrF4CD2v668SAXYHn06tG4ZIbHPpLkp7r7E1X1pCS3VdWto/ve3t2/OHt5AMyiu/P8b/y5Az/uo/e8ZQHVAJMctGd//+43p6oWWNHxdeiV4O5+oLs/Mbr+5SR/kuTyeRXGatveR3bavwDA/g7zofUwH3K5aC77BFfVFUm+Pcl/Gw29saruqKrrq+op83gNVsvJEycO9BcAYJXMnFCq6olJbkry4939pSTvSPLsJKeTPJDkbXs87kxVnauqc7PWwNGzEjw84z17/vz5ZZcD7EO/wmQzheCq+ppcDMC/1t03J0l3P9jdF7p7K8k7k1y122O7+2x3b3b35iw1sBxWgodnvGc3NjaWXQ6wD/0Kkx36h3F1cS/sdyX5k+7+t2Pjl3X3A6Obr0xy52wlsooubG3l5IkTU/8FmIeD/Bp+mm39oAgWZ9X7dZajQ/yTJD+c5I+r6vbR2JuTXFNVp5N0knuTXDdThayk47ISvDVlg1bVxG27e+XfL6yz7QPrT2uaba99wUdnKQnYwzr066FDcHd/JMlukfyWw5fDujguK8EH+eTpeIwAcHzMshLMgB2XleBp6tsaHZB81d8LHHdVNXElaHz1ySovLM869KtZnUNxdAgAYJ0JwRzKcVkJBgCGSULhUKwEA8B8VVV+/+43H+gxTnF+eEIwh2IlGADm7yBBWACejR/GARxjVWWihDWjb4+GZToAAAZHCAYAYHCEYJjASTIA4PixTzBM4Md9sD6mOUA/sBqW3a9mdwAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHBmPkRaVd2b5MtJLiR5pLs3q+qpSW5MckWSe5O8pru/OOtrAQDAPMxrJfi7u/t0d2+Obr8pyYe6+8okHxrdBgCAlbCo3SFekeQ9o+vvSfJDC3odAAA4sHmE4E7ywaq6rarOjMYu7e4HRtf/PMmlc3gdAACYi3mcNvk7u/v+qvr7SW6tqk+P39ndXVW980GjwHxm5ziwmsZ79tSpU0uuBtiPfoXJZl4J7u77R38fSvL+JFclebCqLkuS0d+Hdnnc2e7eHNuPGFhh4z27sbGx7HKAfehXmGymEFxVT6iqJ21fT/K9Se5M8oEk1442uzbJb83yOgAAME+z7g5xaZL3V9X2c/16d/+Xqvp4kvdV1euSfDbJa2Z8HQAAmJuZQnB3fybJc3cZ/0KSF8/y3AAAsCjz+GEcAMxN9+N+S/04o28ggSVb534Vgjn2tiY0aHfn5AlnEIdl255Mr7v58xO3/dVXfUOS1Z1c4bg7Dv0qBHNsTQq/26oqW93CMCzJQSbTbdvbrurkCsfVcepXIZhjZ7/wO/61zc4mFIbh6HX3gSbTncYn11WZWOG4Om79KgRzrOwWgMeD73i4vbC19ej18WasqlzY2hKEYcGmmVDPvvryqba97ubPr8zECsfRcexXIZhjY68AvFeY3RmIBWE4OvtNktsT6biqmjjBXnfz53d9LDCb49qvZniOrYPs1nDyxImpfuEKLNY0k2JVPbpv4U76GI7OuverEMyxsHMV+DD79e4MwturwcB87bUydJBVob0m1ln2VwQe7zj3qxDMsTPLD9usCMNyHOZr0b0mVj0Mi3Vc+lUIZu2NrwLP48gO44+3Ggzztduq0iz7Be42sS57dQmOi+Per0IwAACDIwTDLnydCgDHmxDMsTHPk1w4NBocjXkcImm/X58D83Pc+tVMDwDA4AjBACyNXY9gfRy3fhWCOTbmeSSH3c4+B8zfPH4ZPs3pXIHZHbd+PfRpk6vqm5PcODb0rCT/KsmTk/zzJOdH42/u7lsOXSEAAMzZoUNwd9+V5HSSVNXJJPcneX+SH03y9u7+xblUCBN0d6pq2WUAh6SHYX0cp36d1+4QL05yT3d/dk7PB1Ob98kt5n3yDeCr9jpY/mH3NZz3wfyBrzru/Tqv2f3qJDeM3X5jVd1RVddX1VPm9Bqwp/GGnCUI2xcYluMwE+sq7VsIQ3Jc+nXmEFxVX5vkB5P8h9HQO5I8Oxd3lXggydv2eNyZqjpXVedmrQF2rtYeJgjvDMBWgR9rvGfPnz8/+QGwh72OE3qQiXWvCdUq8EX6lXk5zv06jxn++5N8orsfTJLufrC7L3T3VpJ3Jrlqtwd199nu3uzuzTnUAI9rxoMEYSvAk4337MbGxrLL4Zjanli3L+PGx1dtRWnV6FeOwrr366F/GDfmmoztClFVl3X3A6Obr0xy5xxeAyY6eeJELmxtPWaH/ap6NODuXNndL/haBYbF2l5d2m1yHB8bX4GaNJEue1UJjqvj2q8zheCqekKSlyS5bmz4F6rqdJJOcu+O+2ChtoNwksf9enU8EO9l+5OsAAyLt9/Eum3aFaRVmFDhODuO/TpTCO7uv0ny9TvGfnimimBG2wF2rzC8G+EXlqOqcvbVlx/6K9NVmUxhCI5bv85jdwhYSTvD8DTbAstx0Ml11SZTGJLj0q9CMMeegAvrY3tyBVbfuverdAAAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADM5UIbiqrq+qh6rqzrGxp1bVrVX1Z6O/TxmNV1X9P1V1d1XdUVX/aFHFAwDAYUy7EvzuJC/dMfamJB/q7iuTfGh0O0m+P8mVo8uZJO+YvUwAAJifqUJwd384ycM7hl+R5D2j6+9J8kNj4+/tiz6W5MlVddk8igUAgHmYZZ/gS7v7gdH1P09y6ej65Uk+N7bdfaMxAABYCXP5YVx3d5I+yGOq6kxVnauqc/OoAVis8Z49f/78sssB9qFfYbJZQvCD27s5jP4+NBq/P8kzxrZ7+mjsMbr7bHdvdvfmDDUAR2S8Zzc2NpZdDrAP/QqTzRKCP5Dk2tH1a5P81tj4j4yOEvEdSf5qbLcJAABYukum2aiqbkjywiRPq6r7krw1yc8neV9VvS7JZ5O8ZrT5LUleluTuJH+b5EfnXDMAAMxkqhDc3dfscdeLd9m2k7xhlqIAAGCRnDEOAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGJyJIbiqrq+qh6rqzrGxf1NVn66qO6rq/VX15NH4FVX1d1V1++jyK4ssHgAADmOaleB3J3npjrFbk3xbd//DJH+a5KfH7runu0+PLq+fT5kAADA/E0Nwd384ycM7xj7Y3Y+Mbn4sydMXUBsAACzEPPYJ/rEk/3ns9jOr6g+r6ner6gVzeH4AAJirS2Z5cFW9JckjSX5tNG4GsWoAAB3RSURBVPRAklPd/YWqel6S36yq53T3l3Z57JkkZ2Z5feDojPfsqVOnllwNsB/9CpMdeiW4ql6b5AeS/LPu7iTp7q909xdG129Lck+Sb9rt8d19trs3u3vzsDUAR2e8Zzc2NpZdDrAP/QqTHWoluKpemuRfJvnfu/tvx8Y3kjzc3Req6llJrkzymblUCsDcjNYudlVVR1gJMIl+XYyJIbiqbkjywiRPq6r7krw1F48G8XVJbh39439sdCSI70ryr6vqfybZSvL67n541yfmWNnao0G7OydPOBw1i7Pf5LCTyeKi7s5f/9HJPe9/4nMv+LdiIfTrwenXxZkYgrv7ml2G37XHtjcluWnWopZprzC3k3D3Vfv9m1VVLmxt+bdiIbo7H//OF029/VW/9zsLrGY9TJpQk+Sv/+ikiZW5068Hp18XSzJhJtN8aNgOwjAUB1ntOkrTTKjbpt0OWAz9unhC8BoQIGF9dHdOnH35ygZh4LG6W78OlBC8BqykwvoRhGH1bX9o1a/DJASvCUEY1o+JFdaHfh0eIXiNCMKwfkyssD7067AIwWtGEIb1Y2KF9aFfh0MIXkOrFIT9jwKmY2KF9aFfh0EIXlOrEoRPnjgx8X8UjqkMFy17Yq2qPPG5F6ba9kmnl///F1gm/Xr8SSZrbNWC8F4XARi+ah0mVhMqXKRfj7eJZ4wbmnX7+mNVzsi27NeHdXLi7MuzdeY/Le0MT1Vl4oQp6dfjSwjeYR3D3KoEYThqVbW2p1Zd9sQKR02/smqkpmNiVXaNAKa37K9agenp1+NHCD5GBGFYPyZWWB/69XgRgo8ZQRjWj4kV1od+PT6E4GNIEIb1Y2KF9aFfjwch+JgShGH9mFhhfejX9TcxBFfV9VX1UFXdOTb2M1V1f1XdPrq8bOy+n66qu6vqrqr6vkUVzmSCMKwfEyusjxNnX77sEpjBNCvB707y0l3G397dp0eXW5Kkqr41ydVJnjN6zL+vqpPzKnao9jsRxaRLEkEY1oyJFWDxJh4nuLs/XFVXTPl8r0jyG939lST/varuTnJVko8eukIc/xfWSFWlr/vtZZcBTEG/Dtss6eqNVXXHaHeJp4zGLk/yubFt7huNAQDAyjhsCH5HkmcnOZ3kgSRvO+gTVNWZqjpXVecOWQNwhMZ79vz588suB9iHfoXJDhWCu/vB7r7Q3VtJ3pmLuzwkyf1JnjG26dNHY7s9x9nu3uzuzcPUAByt8Z7d2NhYdjnAPvQrTHaoEFxVl43dfGWS7SNHfCDJ1VX1dVX1zCRXJvmD2UoEAID5mvjDuKq6IckLkzytqu5L8tYkL6yq00k6yb1JrkuS7v5kVb0vyaeSPJLkDd19YTGlAwDA4UxzdIhrdhl+1z7b/2ySn52lKAAAWCTH3gIAYHCEYAAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHCEYAAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHCEYAAABkcIBgBgcIRgAAAGZ2IIrqrrq+qhqrpzbOzGqrp9dLm3qm4fjV9RVX83dt+vLLJ4AAA4jEum2ObdSX45yXu3B7r7n25fr6q3Jfmrse3v6e7T8yoQAADmbWII7u4PV9UVu91XVZXkNUleNN+yAABgcWbdJ/gFSR7s7j8bG3tmVf1hVf1uVb1gxucHAIC5m2Z3iP1ck+SGsdsPJDnV3V+oqucl+c2qek53f2nnA6vqTJIzM74+cETGe/bUqVNLrgbYj36FyQ69ElxVlyR5VZIbt8e6+yvd/YXR9duS3JPkm3Z7fHef7e7N7t48bA3A0Rnv2Y2NjWWXA+xDv8Jks+wO8T1JPt3d920PVNVGVZ0cXX9WkiuTfGa2EgEAYL6mOUTaDUk+muSbq+q+qnrd6K6r89hdIZLku5LcMTpk2n9M8vrufnieBQMAwKymOTrENXuMv3aXsZuS3DR7WQAAsDjOGAcAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAxOdfeya0hVnU/yN0n+Ytm1zMHT4n2sknV4H/+guzeWXcRBVNWXk9y17DrmYB3++5iG93G01qpn9evK8T6O1p79eslRV7Kb7t6oqnPdvbnsWmblfayW4/I+VtBdx+Hf9bj89+F9MIF+XSHex+qwOwQAAIMjBAMAMDirFILPLruAOfE+VstxeR+r5rj8u3ofq+W4vI9Vc1z+Xb2P1bL272MlfhgHAABHaZVWggEA4EgIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgLCwEV9VLq+quqrq7qt60qNcBAICDqu6e/5NWnUzyp0lekuS+JB9Pck13f2ruLwYAAAe0qJXgq5Lc3d2f6e7/keQ3krxiQa8FAAAHcsmCnvfyJJ8bu31fkn88vkFVnUlyZnTzeQuqA9bBX3T3xrKLmGS8Z5/whCc871u+5VuWXBEsx2233bbyPatf4aL9+nVRIXii7j6b5GySVNX898mA9fHZZRcwjfGe3dzc7HPnzi25IliOqlr5ntWvcNF+/bqo3SHuT/KMsdtPH40BAMDSLSoEfzzJlVX1zKr62iRXJ/nAgl4LAAAOZCG7Q3T3I1X1xiT/NcnJJNd39ycX8VoAAHBQC9snuLtvSXLLop4fAAAOyxnjAAAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgME5dAiuqmdU1e9U1aeq6pNV9S9G4z9TVfdX1e2jy8vmVy4AAMzukhke+0iSn+ruT1TVk5LcVlW3ju57e3f/4uzlAQDA/B06BHf3A0keGF3/clX9SZLL51UYAAAsylz2Ca6qK5J8e5L/Nhp6Y1XdUVXXV9VT9njMmao6V1Xn5lEDsFjjPXv+/PlllwPsQ7/CZDOH4Kp6YpKbkvx4d38pyTuSPDvJ6VxcKX7bbo/r7rPdvdndm7PWACzeeM9ubGwsuxxgH/oVJpspBFfV1+RiAP617r45Sbr7we6+0N1bSd6Z5KrZywQAgPmZ5egQleRdSf6ku//t2PhlY5u9Msmdhy8PAADmb5ajQ/yTJD+c5I+r6vbR2JuTXFNVp5N0knuTXDdThQAAMGezHB3iI0lql7tuOXw5AACweM4YBwDA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAyOEAwAwOAIwQAADI4QDADA4AjBAAAMjhAMAMDgCMEAAAzOJbM+QVXdm+TLSS4keaS7N6vqqUluTHJFknuTvKa7vzjrawEAwDzMayX4u7v7dHdvjm6/KcmHuvvKJB8a3QYAgJWwqN0hXpHkPaPr70nyQwt6HQAAOLB5hOBO8sGquq2qzozGLu3uB0bX/zzJpTsfVFVnqupcVZ2bQw3Ago337Pnz55ddDrAP/QqTzSMEf2d3/6Mk35/kDVX1XeN3dnfnYlDOjvGz3b05tgsFsMLGe3ZjY2PZ5QD70K8w2cwhuLvvH/19KMn7k1yV5MGquixJRn8fmvV1AABgXmYKwVX1hKp60vb1JN+b5M4kH0hy7Wiza5P81iyvAwAA8zTrIdIuTfL+qtp+rl/v7v9SVR9P8r6qel2SzyZ5zYyvAwAAczNTCO7uzyR57i7jX0jy4lmeGwAAFsUZ4wCOse7Oxd8nA6tOvx4tIZgD2erOha2tZZcBTKG7896PPD/v/cjzl10KMIF+PXpCMAAAgyMEAwAwOEIwAACDIwQDADA4QjAAAIMz68kyAFiyaQ6ptN82oxMeAUdAv64OIZjH2ZrQoFW15zbdnZMnfMEAR2X7sEqT7LfNtS/46DxLAvagX1eLEMzjTPMJ1MG8AYB1JgTzOPut5G6NzmZjtRdWQ1XtuTI0vupk9QiWT7+uFkkGAIDBsRIMHLlpdqfx4w9YDfqV4+pYhOBJP+Ty9T2sju7OX//RyYnbPfG5F0yssGT6leNs7ZPhpACcXPyEemFr6wiqAfbSo/3Jp5lQk+Sv/+jko48BjpZ+ZQjWdiV4mvA7bvuwXlaF4egdZDIdt/2YJ532IRaOin5lKA4dgqvqm5PcODb0rCT/KsmTk/zzJOdH42/u7lsOXSErxad8lqG7fdUKa0K/si4OHYK7+64kp5Okqk4muT/J+5P8aJK3d/cvzqXCXRx0FXjc9q4RVoMPx78bB3XYVaVxf/1HJ+1zeAj7HY4JdqNfl0e/Hr15JZoXJ7mnuz87p+fb0ywBeJt9hOFozGNC3Tav59nL9v6M01zgONKvDM289gm+OskNY7ffWFU/kuRckp/q7i/ufEBVnUlyZk6vv3KmDev2UWZdjPfsqVOnllzN/GxPktfd/PmpH/Orr/qGJA4LxerSr1+lX9lLzfopqaq+Nsnnkzynux+sqkuT/EWSTvJ/J7msu39swnNMXcQ8VoKTxYXPw9YnDA/abd29uewiDmJzc7PPnTs3cbt5riwl8//BTXcfaDLd6Vdf9Q0m1gGqqrXqWf16kX4dpv36dR6p6/uTfKK7H0yS7n6wuy9091aSdya5ag6vsRbmsa8ycDRmnVCTi6tRvm6FxdOvLMI8QvA1GdsVoqouG7vvlUnunMNrrLy9AvBB9lMShOFozGNC3WZihcXSryzKTPsEV9UTkrwkyXVjw79QVadzcXeIe3fcdyztFoC3m2y/XRy2A+/41zOOXgGLNWlCPfvqyw/8uOtu/ryvWmEB9CuLNFMI7u6/SfL1O8Z+eKaK1sxeAXiaELu9zYWtLUEYjsB+E+Nek+m2qsrZV1++53OYWGG+9CuLJmXNYJYAPO7kiROP+3rGrhEwX7NMqOOq6tFfm++08/kPchgnh3WCr1pGv+58fX16/K3daZNX+Uw0sxzh4eSJE49bEQYW7yAT6rbtiXW3SXT7/1Hdned/488duq6P3vOWQz8Wlm1Rp05eVL+O304yde/+/t1vfvQ1WD9rtxK826rpQc3jcGQ7V4Hn8Zw7H7/bavD27YP+hWWoqjzxuRfm8lyzTKp7rSodZkLdttcK07x+wANHbZ79Ootl9ev2B9eDfHg96PaslrULwclsp+5d9ePxTgr427Uf9C8syzwm1kWtKgGPNdR+nfWbG9bT2iakw+yPs6j9d+YZrCc9j5VgmI9ZVpW27be/IQzRKu0GsZN+Zae12yd4215HVtjLqq8AT8tKMOuoqvKk01sHPiPVOq4owbrTrwzF2iek7X2EJ12OSxi0Esw6O8hXrSZUWC79ynG3tivB445LwJ2GlWDW3fYqE7D69CvHmYS0ZqwEAwDMTgheM1aCATjO/ICNoyIhrRkrwTAfzvYEK26sR/UriyAEz8E8T3G826mYx1kJhvmYx0kt9ju1KzCbb//Y+x+9rl9ZBAlpzVgJhoumOSrMY44n3v3Vy9hzAIt3kH7t7vzBd74oOw9+ql+Zt2NxdIh5mLQCO8n2avAsK6/TnIrZSjBc7I2Pf+eLDvSY1yX5w//tVenR3+Ti6tKvvuobpjrW+G41zPvUrnAcHaZfK8m3f/TmfOI7XpmM+lO/Mm8S0hzNslvEtCHcSjDMYJc+u+7mzx/q7JO+VoXFqiSve/sPP2ZsUf1aVfn9u9980BKTJB+95y2HehzLJwTP2WGC8G4BeK8TfFgJhsOpXNzHcLc1pO2JddLkur3NXhOqVSVYvEX162GCsAC83qbaHaKqrk/yA0ke6u5vG409NcmNSa5Icm+S13T3F+vi9xT/LsnLkvxtktd29yfmX/rq6O7HfD1TVdma4kx1e63+Hqcz3MEqefQr1tHuEOO2J8r9Ds2032rSbhOqCRJmU7nYkzt7b979+ujr6dtBmXaf4Hcn+eUk7x0be1OSD3X3z1fVm0a3/88k35/kytHlHyd5x+jvsbQdWC9sbT1uP6XtMHyY5wMWY69JddthdnOwAgyLs33c4N16U78yi6nSVnd/OMnDO4ZfkeQ9o+vvSfJDY+Pv7Ys+luTJVXXZPIpdZSdPnJj5l6sCMByNeR6M34QKi/O/fuT/TTK/ntWvjJslcV3a3Q+Mrv95kktH1y9P8rmx7e4bjT1GVZ2pqnNVdW6GGlbKdhA+zE77AjCrbrxnz58/v+xyZjaPSdWEyqo6Lv06flSJWXtWv7LTXA6R1t1dVQdKft19NsnZJDnoY1fZdpA9yI/jhF/WwXjPbm5uHoueraqcffXlBz7ag8mUVXcc+zX5as8m0x31Qa+yn1lC8INVdVl3PzDa3eGh0fj9SZ4xtt3TR2ODItjC+hifWIH1oG+Z1SxJ7QNJrh1dvzbJb42N/0hd9B1J/mpstwkAAFi6aQ+RdkOSFyZ5WlXdl+StSX4+yfuq6nVJPpvkNaPNb8nFw6PdnYuHSPvROdcMsBQ7D4cIwPqaKgR39zV73PXiXbbtJG+YpaihmPU0y8DR6e6cOPvybJ35T4IwrIHtH6nrV/YigS3RLKdZBpbjxNmXz3w4RGCxtj+06lf2IwQvmSAM68fECutDv7IXIXhk+1i9B73MgyAM68fECutDv7KbuRwn+DhY9r6520F42XXAOqiqXPV7v7PsMuwjDFPQr6wqiWuFWBGG9WOFCdaHfmWcELxiBGFYPyZWWB/6lW1C8AoShGH9mFhhfehXEiF4ZQnCsH5MrLA+9CtC8AoThGH9mFhhfejXYROCV5wgDOvHxArrQ78OlxC8BgRhWD8mVlgf+nWYhOA1IQjD+jGxwvrQr8MjBK8RQRjWj4kV1od+HRYheM0IwrB+TKywPvTrcAjBa0gQhvVjYoX1oV+HQQheU4IwrB8TK6wP/Xr8TQzBVXV9VT1UVXeOjf2bqvp0Vd1RVe+vqiePxq+oqr+rqttHl19ZZPFDJwjD+jGxwvrQr8fbNCvB707y0h1jtyb5tu7+h0n+NMlPj913T3efHl1eP58y2YsgDOvHxArr48TZly+7BBZkYgju7g8neXjH2Ae7+5HRzY8lefoCajv2unsulySCMKwZEyvAcl0yh+f4sSQ3jt1+ZlX9YZIvJfm/uvv/2+1BVXUmyZk5vP7aOnnCLtmsj/GePXXq1JKrOXpVlb7ut5ddBkxFv+pXJpsphVXVW5I8kuTXRkMPJDnV3d+e5CeT/HpV/b3dHtvdZ7t7s7s3Z6kBOBrjPbuxsbHscoB96FeY7NAhuKpem+QHkvyzHn0n391f6e4vjK7fluSeJN80hzoBAGBuDhWCq+qlSf5lkh/s7r8dG9+oqpOj689KcmWSz8yjUAAAmJeJ+wRX1Q1JXpjkaVV1X5K35uLRIL4uya1VlSQfGx0J4ruS/Ouq+p9JtpK8vrsf3vWJAQBgSSaG4O6+Zpfhd+2x7U1Jbpq1KAAAWCSHJwAAYHCEYAAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHCEYAAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHCEYAAABkcIBgBgcIRgAAAGZ2IIrqrrq+qhqrpzbOxnqur+qrp9dHnZ2H0/XVV3V9VdVfV9iyocAAAOa5qV4Hcneeku42/v7tOjyy1JUlXfmuTqJM8ZPebfV9XJeRULAADzMDEEd/eHkzw85fO9IslvdPdXuvu/J7k7yVUz1AcAAHM3yz7Bb6yqO0a7SzxlNHZ5ks+NbXPfaOxxqupMVZ2rqnMz1AAckfGePX/+/LLLAfahX2Gyw4bgdyR5dpLTSR5I8raDPkF3n+3uze7ePGQNwBEa79mNjY1llwPsQ7/CZIcKwd39YHdf6O6tJO/MV3d5uD/JM8Y2ffpoDAAAVsahQnBVXTZ285VJto8c8YEkV1fV11XVM5NcmeQPZisRAADm65JJG1TVDUlemORpVXVfkrcmeWFVnU7SSe5Ncl2SdPcnq+p9ST6V5JEkb+juC4spHQAADmdiCO7ua3YZftc+2/9skp+dpSgAAFgkZ4wDAGBwhGAAAAZHCAYAYHCEYAAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHCEYAAABkcIBgBgcIRgAAAGRwgGAGBwhGAAAAZHCAYAYHCEYAAABmdiCK6q66vqoaq6c2zsxqq6fXS5t6puH41fUVV/N3bfryyyeAAAOIxLptjm3Ul+Ocl7twe6+59uX6+qtyX5q7Ht7+nu0/MqEAAA5m1iCO7uD1fVFbvdV1WV5DVJXjTfsgAAYHFm3Sf4BUke7O4/Gxt7ZlX9YVX9blW9YK8HVtWZqjpXVedmrAE4AuM9e/78+WWXA+xDv8Jks4bga5LcMHb7gSSnuvvbk/xkkl+vqr+32wO7+2x3b3b35ow1AEdgvGc3NjaWXQ6wD/0Kkx06BFfVJUleleTG7bHu/kp3f2F0/bYk9yT5plmLBACAeZplJfh7kny6u+/bHqiqjao6Obr+rCRXJvnMbCUCAMB8TXOItBuSfDTJN1fVfVX1utFdV+exu0IkyXcluWN0yLT/mOT13f3wPAsGAIBZTXN0iGv2GH/tLmM3Jblp9rIAAGBxnDEOAIDBEYIBABgcIRgAgMERggEAGBwhGACAwRGCAQAYHCEYAIDBEYIBABgcIRgAgMERggEAGBwhGACAwanuXnYNqarzSf4myV8su5Y5eFq8j1WyDu/jH3T3xrKLOIiq+nKSu5Zdxxysw38f0/A+jtZa9ax+XTnex9Has18vOepKdtPdG1V1rrs3l13LrLyP1XJc3scKuus4/Lsel/8+vA8m0K8rxPtYHXaHAABgcIRgAAAGZ5VC8NllFzAn3sdqOS7vY9Ucl39X72O1HJf3sWqOy7+r97Fa1v59rMQP4wAA4Cit0kowAAAciaWH4Kp6aVXdVVV3V9X/3979vNgUxnEcf39SLKT8WEhYoNmwmSRZSNlgbIbd2LBQNvwBZOMfkFJYKA0W7MTCArOxkh/l58JvxYRZKEvE1+I8k2PMwR25z3nmfl51uueee6c+zzn3U0/3nHNnf+48nZD0StIDSXcl3U7b5ku6KulpepyXO+dEkk5JGpP0sLZt0tyqHE3H576k1fmS/6xhHIckjaZjclfS1tprB9I4HkvanCd1+dzZ7nNn3dmpcl+7z30tp69ZJ8GSZgDHgAFgJbBD0sqcmaZgY0T0134mZD8wEhF9wEh63jbDwJYJ25pyDwB9adkDnOhSxr8xzK/jADiSjkl/RFwGSJ+rIWBV+pvj6fNnHXBnsxnGnXVnO+S+ZjOM+1pEX3N/E7wWeBYRLyLiM3AeGMyc6V8NAqfT+mlgW8Ysk4qI68CHCZubcg8CZ6JyA5graVF3kv5ewziaDALnI+JTRLwEnlF9/qwz7mwG7qw7O0Xuawbuazl9zT0JXgy8rj1/k7aVIoArku5I2pO2LYyIt2n9HbAwT7SONeUu8RjtS6eVTtVOlZU4jjYqfT+6s+3kzv4fpe9D97Wdpk1fc0+CS7c+IlZTnc7YK2lD/cWofnqjuJ/fKDV3cgJYAfQDb4HDeeNYy7iz7ePOWhP3tX2mVV9zT4JHgaW150vStiJExGh6HAMuUH31/378VEZ6HMuXsCNNuYs6RhHxPiK+RsQ34CQ/TscUNY4WK3o/urPt487+V0XvQ/e1faZbX3NPgm8BfZKWSZpJdVH1pcyZ/oqk2ZLmjK8Dm4CHVPl3pbftAi7mSdixptyXgJ3pDtZ1wMfaKZ3WmXAt1XaqYwLVOIYkzZK0jOomhJvdzjcNuLPt4c7an7iv7eG+tlFEZF2ArcAT4DlwMHeeDnIvB+6l5dF4dmAB1Z2fT4FrwPzcWSfJfo7qNMYXqut2djflBkR1d/Fz4AGwJnf+P4zjbMp5n6qUi2rvP5jG8RgYyJ2/1MWdzZLdnXVnp7rP3dfuZ3dfC+mr/2OcmZmZmfWc3JdDmJmZmZl1nSfBZmZmZtZzPAk2MzMzs57jSbCZmZmZ9RxPgs3MzMys53gSbGZmZmY9x5NgMzMzM+s5ngSbmZmZWc/5DiDDyotwaZrwAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 864x864 with 9 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wsfxcw0-DZdn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JPG4VNTnFr3p",
        "colab_type": "text"
      },
      "source": [
        "## Next steps\n",
        "\n",
        "Try tweaking the hyper-parameters for better accuracy e.g.\n",
        "\n",
        "- learning rates and schedules\n",
        "- loss weights\n",
        "- unfreezing layers\n",
        "- batch size\n",
        "- etc."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7VHV2fS4GRd-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 15,
      "outputs": []
    }
  ]
}