BERT_Cover_Classifier.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "BERT Cover Classifier",
      "provenance": [],
      "collapsed_sections": [],
      "mount_file_id": "1RxV8QtZYMGDzhIDvfDpFF6twroXbm56T",
      "authorship_tag": "ABX9TyOdbbrq2E6C3TVUfv1yuBia",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "24ce0dc800e647f195a4bb3d7a4dbc45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_63cc63a2dc2b495ab8d99a832544ddbb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a9b86c78368d4fa4a4568c564a607f21",
              "IPY_MODEL_8dacec2ee163488baabd3ffa1583ae1c"
            ]
          }
        },
        "63cc63a2dc2b495ab8d99a832544ddbb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a9b86c78368d4fa4a4568c564a607f21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_00f7b1b2795c4ae1a17ddffa24d74f64",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 433,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 433,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_0de6b48444be482a947414c6f3abb2ab"
          }
        },
        "8dacec2ee163488baabd3ffa1583ae1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2138c62d78bf4daa965444b28707ee0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 433/433 [00:07&lt;00:00, 58.0B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_328c80e26f70410c84112564e189b88e"
          }
        },
        "00f7b1b2795c4ae1a17ddffa24d74f64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "0de6b48444be482a947414c6f3abb2ab": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2138c62d78bf4daa965444b28707ee0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "328c80e26f70410c84112564e189b88e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "09debd7a132f40c2a5c9d831de72e9ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_23d8892dad374ab592c9f1cbf04b56e6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_ab196dbb95b04e578f2cf1e3b66cdce7",
              "IPY_MODEL_dfe2cb3bf4344f069335238bfd2d10bd"
            ]
          }
        },
        "23d8892dad374ab592c9f1cbf04b56e6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ab196dbb95b04e578f2cf1e3b66cdce7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a72fb5def1a84f65bb56dafd708e9dcd",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 440473133,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 440473133,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_759fa41b1bc846cca3055136eaf48e7a"
          }
        },
        "dfe2cb3bf4344f069335238bfd2d10bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_878b194ac37049e080a2c23d83b8140a",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 440M/440M [00:06&lt;00:00, 68.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_91c08f90e4b24f398f8a4c56f8fd5afe"
          }
        },
        "a72fb5def1a84f65bb56dafd708e9dcd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "759fa41b1bc846cca3055136eaf48e7a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "878b194ac37049e080a2c23d83b8140a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "91c08f90e4b24f398f8a4c56f8fd5afe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4bac85dbe578468ebcdb49242d9d155c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ed8dc26c014048099c7849b27a68fc73",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_3a62c708eb674285a3b04f58ee8e78bc",
              "IPY_MODEL_8613daaf77a3463ba683b0de984ed72f"
            ]
          }
        },
        "ed8dc26c014048099c7849b27a68fc73": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "3a62c708eb674285a3b04f58ee8e78bc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5ee038bee91141d4a736ab305919ecbd",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a35f06e481f14df985b0c3886bef25ae"
          }
        },
        "8613daaf77a3463ba683b0de984ed72f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f3ded66ed888426c9212737738809f23",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:20&lt;00:00, 11.4kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_05fd555f6e614253a771ee4fbff7a69d"
          }
        },
        "5ee038bee91141d4a736ab305919ecbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a35f06e481f14df985b0c3886bef25ae": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f3ded66ed888426c9212737738809f23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "05fd555f6e614253a771ee4fbff7a69d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/SeaOfFrost/BookCoverClassifier/blob/master/BERT_Cover_Classifier.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1iy3lDxpzu24",
        "colab_type": "text"
      },
      "source": [
        "# Load the Required Folders"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GyZfC2Tvk6g0",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "f4caa328-525c-4375-9860-121b860e5947"
      },
      "source": [
        "ls"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[0m\u001b[01;34mdrive\u001b[0m/  \u001b[01;34msample_data\u001b[0m/\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T8kVe2tjl0Do",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "0ef30b31-a2f8-499e-c904-213fd49ea8f1"
      },
      "source": [
        "cd drive/My Drive/Book Classifier/book-dataset/Task1/"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/content/drive/My Drive/Book Classifier/book-dataset/Task1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "UN2jq6s40Am0",
        "colab_type": "text"
      },
      "source": [
        "## Install required dependencies"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_7-l51dmkpA",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 655
        },
        "outputId": "0b6b50bb-99f2-4211-9908-be297777f920"
      },
      "source": [
        "!pip install transformers"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/27/3c/91ed8f5c4e7ef3227b4119200fc0ed4b4fd965b1f0172021c25701087825/transformers-3.0.2-py3-none-any.whl (769kB)\n",
            "\u001b[K     |████████████████████████████████| 778kB 7.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 22.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Collecting tokenizers==0.8.1.rc1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/40/d0/30d5f8d221a0ed981a186c8eb986ce1c94e3a6e87f994eae9f4aa5250217/tokenizers-0.8.1rc1-cp36-cp36m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 34.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d4/a4/d0a884c4300004a78cca907a6ff9a5e9fe4f090f5d95ab341c53d28cbc58/sentencepiece-0.1.91-cp36-cp36m-manylinux1_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 56.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893260 sha256=5caec7a1999982c33669eba743722825ea353c8c26bb4024a953ef946870d1d3\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.91 tokenizers-0.8.1rc1 transformers-3.0.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZM1qbZtJm4r0",
        "colab_type": "text"
      },
      "source": [
        "# Load Dependencies\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V3qS-xvDm3OE",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.metrics import classification_report\n",
        "import transformers\n",
        "from transformers import AutoModel, BertTokenizerFast"
      ],
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a7yTrH8gnAU4",
        "colab_type": "text"
      },
      "source": [
        "# Load the Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QWZyrhBXm-sM",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 310
        },
        "outputId": "1d1dfdca-08d7-47f0-e18e-ff91745868ff"
      },
      "source": [
        "df_train = pd.read_csv(\"train.csv\", names = [\"ASIN\",\"FILENAME\",\"IMAGE_URL\",\"TITLE\",\"AUTHOR\",\"CATEGORY ID\",\"CATEGORY\"], engine = \"python\")\n",
        "df_train.head()"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ASIN</th>\n",
              "      <th>FILENAME</th>\n",
              "      <th>IMAGE_URL</th>\n",
              "      <th>TITLE</th>\n",
              "      <th>AUTHOR</th>\n",
              "      <th>CATEGORY ID</th>\n",
              "      <th>CATEGORY</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1588345297</td>\n",
              "      <td>1588345297.jpg</td>\n",
              "      <td>http://ecx.images-amazon.com/images/I/51l6XIoa...</td>\n",
              "      <td>With Schwarzkopf: Life Lessons of The Bear</td>\n",
              "      <td>Gus Lee</td>\n",
              "      <td>1</td>\n",
              "      <td>Biographies &amp; Memoirs</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1404803335</td>\n",
              "      <td>1404803335.jpg</td>\n",
              "      <td>http://ecx.images-amazon.com/images/I/51UJnL3T...</td>\n",
              "      <td>Magnets: Pulling Together, Pushing Apart (Amaz...</td>\n",
              "      <td>Natalie M. Rosinsky</td>\n",
              "      <td>4</td>\n",
              "      <td>Children's Books</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>1446276082</td>\n",
              "      <td>1446276082.jpg</td>\n",
              "      <td>http://ecx.images-amazon.com/images/I/51MGUKhk...</td>\n",
              "      <td>Energy Security (SAGE Library of International...</td>\n",
              "      <td>NaN</td>\n",
              "      <td>10</td>\n",
              "      <td>Engineering &amp; Transportation</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>1491522666</td>\n",
              "      <td>1491522666.jpg</td>\n",
              "      <td>http://ecx.images-amazon.com/images/I/51qKvjsi...</td>\n",
              "      <td>An Amish Gathering: Life in Lancaster County</td>\n",
              "      <td>Beth Wiseman</td>\n",
              "      <td>9</td>\n",
              "      <td>Christian Books &amp; Bibles</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>970096410</td>\n",
              "      <td>0970096410.jpg</td>\n",
              "      <td>http://ecx.images-amazon.com/images/I/51qoUENb...</td>\n",
              "      <td>City of Rocks Idaho: A Climber's Guide (Region...</td>\n",
              "      <td>Dave Bingham</td>\n",
              "      <td>26</td>\n",
              "      <td>Sports &amp; Outdoors</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "         ASIN        FILENAME  ... CATEGORY ID                      CATEGORY\n",
              "0  1588345297  1588345297.jpg  ...           1         Biographies & Memoirs\n",
              "1  1404803335  1404803335.jpg  ...           4              Children's Books\n",
              "2  1446276082  1446276082.jpg  ...          10  Engineering & Transportation\n",
              "3  1491522666  1491522666.jpg  ...           9      Christian Books & Bibles\n",
              "4   970096410  0970096410.jpg  ...          26             Sports & Outdoors\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NcZ5uByTnPK4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "15bb4ee5-cda9-4509-ab84-f8124291a7d0"
      },
      "source": [
        "df_train.shape"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(51300, 7)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9NJ6Y-QQp5_s",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 580
        },
        "outputId": "6ff75dc4-e91f-44c8-efdc-70ac5818d03a"
      },
      "source": [
        "# check class distribution\n",
        "df_train['CATEGORY ID'].value_counts(normalize = True)"
      ],
      "execution_count": 55,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "29    0.033333\n",
              "28    0.033333\n",
              "1     0.033333\n",
              "2     0.033333\n",
              "3     0.033333\n",
              "4     0.033333\n",
              "5     0.033333\n",
              "6     0.033333\n",
              "7     0.033333\n",
              "8     0.033333\n",
              "9     0.033333\n",
              "10    0.033333\n",
              "11    0.033333\n",
              "12    0.033333\n",
              "13    0.033333\n",
              "14    0.033333\n",
              "15    0.033333\n",
              "16    0.033333\n",
              "17    0.033333\n",
              "18    0.033333\n",
              "19    0.033333\n",
              "20    0.033333\n",
              "21    0.033333\n",
              "22    0.033333\n",
              "23    0.033333\n",
              "24    0.033333\n",
              "25    0.033333\n",
              "26    0.033333\n",
              "27    0.033333\n",
              "0     0.033333\n",
              "Name: CATEGORY ID, dtype: float64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 55
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "MCk_SI9xrM58",
        "colab_type": "text"
      },
      "source": [
        "Split into Train and Validation Set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O-lZQrBXrMg8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "train_text, valid_text, train_labels, valid_labels = train_test_split(df_train['TITLE'], df_train['CATEGORY ID'], \n",
        "                                                                    random_state=2018, \n",
        "                                                                    test_size=0.1, \n",
        "                                                                    stratify=df_train['CATEGORY ID'])"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IVNPMJpQqIt4",
        "colab_type": "text"
      },
      "source": [
        "Load the Test Dataset"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FshIgiNlqLWo",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "df_test = pd.read_csv(\"test.csv\", names = [\"ASIN\",\"FILENAME\",\"IMAGE_URL\",\"TITLE\",\"AUTHOR\",\"CATEGORY ID\",\"CATEGORY\"], engine = \"python\")\n",
        "test_text, test_labels = df_test[\"TITLE\"], df_test[\"CATEGORY ID\"]"
      ],
      "execution_count": 7,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XDxOb6zs0Sy4",
        "colab_type": "text"
      },
      "source": [
        "# Initialize and Load the Model"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a20SJtygsyAQ",
        "colab_type": "text"
      },
      "source": [
        "Import BERT and Tokenizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8O8uKYbRuckk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "24ce0dc800e647f195a4bb3d7a4dbc45",
            "63cc63a2dc2b495ab8d99a832544ddbb",
            "a9b86c78368d4fa4a4568c564a607f21",
            "8dacec2ee163488baabd3ffa1583ae1c",
            "00f7b1b2795c4ae1a17ddffa24d74f64",
            "0de6b48444be482a947414c6f3abb2ab",
            "2138c62d78bf4daa965444b28707ee0f",
            "328c80e26f70410c84112564e189b88e",
            "09debd7a132f40c2a5c9d831de72e9ab",
            "23d8892dad374ab592c9f1cbf04b56e6",
            "ab196dbb95b04e578f2cf1e3b66cdce7",
            "dfe2cb3bf4344f069335238bfd2d10bd",
            "a72fb5def1a84f65bb56dafd708e9dcd",
            "759fa41b1bc846cca3055136eaf48e7a",
            "878b194ac37049e080a2c23d83b8140a",
            "91c08f90e4b24f398f8a4c56f8fd5afe",
            "4bac85dbe578468ebcdb49242d9d155c",
            "ed8dc26c014048099c7849b27a68fc73",
            "3a62c708eb674285a3b04f58ee8e78bc",
            "8613daaf77a3463ba683b0de984ed72f",
            "5ee038bee91141d4a736ab305919ecbd",
            "a35f06e481f14df985b0c3886bef25ae",
            "f3ded66ed888426c9212737738809f23",
            "05fd555f6e614253a771ee4fbff7a69d"
          ]
        },
        "outputId": "04e36a5d-f585-4de8-cca7-02fdee0a1a90"
      },
      "source": [
        "# import BERT-base pretrained model\n",
        "bert = AutoModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "# Load the BERT tokenizer\n",
        "tokenizer = BertTokenizerFast.from_pretrained('bert-base-uncased')"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "24ce0dc800e647f195a4bb3d7a4dbc45",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=433.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "09debd7a132f40c2a5c9d831de72e9ab",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=440473133.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4bac85dbe578468ebcdb49242d9d155c",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xqh7eg4buhhw",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 55
        },
        "outputId": "0ee3bc2e-4580-4874-8201-416fe36f8d20"
      },
      "source": [
        "# sample data\n",
        "text = [\"this is a bert model tutorial\", \"we will fine-tune a bert model\"]\n",
        "\n",
        "# encode text\n",
        "sent_id = tokenizer.batch_encode_plus(text, padding=True, return_token_type_ids=False)\n",
        "\n",
        "# output\n",
        "print(sent_id)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': [[101, 2023, 2003, 1037, 14324, 2944, 14924, 4818, 102, 0], [101, 2057, 2097, 2986, 1011, 8694, 1037, 14324, 2944, 102]], 'attention_mask': [[1, 1, 1, 1, 1, 1, 1, 1, 1, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "o7gWlKkcusfU",
        "colab_type": "text"
      },
      "source": [
        "Tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EbSlyZkDutr4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 283
        },
        "outputId": "4fd88299-f483-4b6a-9088-ac9d804b409a"
      },
      "source": [
        "# get length of all the messages in the train set\n",
        "seq_len = [len(i.split()) for i in train_text]\n",
        "pd.Series(seq_len).hist(bins = 30)"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<matplotlib.axes._subplots.AxesSubplot at 0x7f18b9bc7550>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        },
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAD4CAYAAAAO9oqkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAU7UlEQVR4nO3dbYyd9X3m8e+1uAkEpzaUaIRsa80qFhHBDYUREKWqxmEXDGRjXtAIZCUm612/WNKSXVYNbNWlmwQt0TZlibbJygpsSBsxoW62WEBCXIdR1NXy5EDDUykOOMUWwW1syDrQpM7+9sX5mxwNY3x8zjyc43w/0mju+3//7zPXnAO+5n6YM6kqJEm/2P7JQgeQJC08y0CSZBlIkiwDSRKWgSQJWLTQAfp1yimn1MqVK3ua++Mf/5gTTzxxbgMNaBQygjln2yjkHIWMYM5e7dix4++r6h1v2FBVI/lxzjnnVK/uv//+nuculFHIWGXO2TYKOUchY5U5ewU8UjP8m+ppIkmSZSBJsgwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAk0cPbUSS5DfgAsLeqzmxj/xX4l8BPge8BH62ql9u264GNwM+A366q+9r4WuAW4Djgi1V1Uxs/DZgEfgXYAXy4qn46m9/kXFt53T09zdt106VznESS+tPLkcGXgLXTxrYBZ1bVrwJ/A1wPkOQM4Arg3W2fzyc5LslxwB8BFwNnAFe2uQCfAW6uqncC++kUiSRpHh2xDKrq28C+aWPfrKqDbfUBYHlbXgdMVtVPqup5YCdwbvvYWVXPtZ/6J4F1SQK8H9jS9r8duGzA70mSdJRm411L/xXw1ba8jE45HLK7jQG8MG38PDqnhl7uKpbu+W+QZBOwCWBsbIypqameAh44cKDnuf24dvXBI0+CN80w1xlnizln1yjkHIWMYM5BDVQGSX4XOAh8ZXbivLmq2gxsBhgfH6+JiYme9puamqLXuf24qtdrBusPn2GuM84Wc86uUcg5ChnBnIPquwySXEXnwvIF7W1RAfYAK7qmLW9jHGb8h8DSJIva0UH3fEnSPOnr1tJ2Z9DvAB+sqle7Nm0Frkjy1naX0CrgIeBhYFWS05K8hc5F5q2tRO4HLm/7bwDu6u9bkST164hlkOQO4P8ApyfZnWQj8N+BtwPbkjyW5H8AVNWTwJ3AU8A3gKur6mftp/6PAfcBTwN3trkAnwD+fZKddK4h3Dqr36Ek6YiOeJqoqq6cYfiw/2BX1Y3AjTOM3wvcO8P4c3TuNpIkLRB/A1mSZBlIkiwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKwDCRJ9FAGSW5LsjfJE11jJyfZluTZ9vmkNp4kn0uyM8l3k5zdtc+GNv/ZJBu6xs9J8njb53NJMtvfpCTpzfVyZPAlYO20seuA7VW1Ctje1gEuBla1j03AF6BTHsANwHnAucANhwqkzfk3XftN/1qSpDl2xDKoqm8D+6YNrwNub8u3A5d1jX+5Oh4AliY5FbgI2FZV+6pqP7ANWNu2/XJVPVBVBXy567EkSfNkUZ/7jVXVi235B8BYW14GvNA1b3cbe7Px3TOMzyjJJjpHHIyNjTE1NdVT2AMHDvQ8tx/Xrj7Y07w3yzDXGWeLOWfXKOQchYxgzkH1Wwavq6pKUrMRpoevtRnYDDA+Pl4TExM97Tc1NUWvc/tx1XX39DRv1/rDZ5jrjLPFnLNrFHKOQkYw56D6vZvopXaKh/Z5bxvfA6zomre8jb3Z+PIZxiVJ86jfMtgKHLojaANwV9f4R9pdRecDr7TTSfcBFyY5qV04vhC4r237UZLz211EH+l6LEnSPDniaaIkdwATwClJdtO5K+gm4M4kG4HvAx9q0+8FLgF2Aq8CHwWoqn1JPgU83OZ9sqoOXZT+t3TuWDoB+Hr7kCTNoyOWQVVdeZhNF8wwt4CrD/M4twG3zTD+CHDmkXJIkuaOv4EsSbIMJEmWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKwDCRJWAaSJAYsgyT/LsmTSZ5IckeS45OcluTBJDuTfDXJW9rct7b1nW37yq7Hub6NP5PkosG+JUnS0eq7DJIsA34bGK+qM4HjgCuAzwA3V9U7gf3AxrbLRmB/G7+5zSPJGW2/dwNrgc8nOa7fXJKkozfoaaJFwAlJFgFvA14E3g9sadtvBy5ry+vaOm37BUnSxier6idV9TywEzh3wFySpKOQqup/5+Qa4EbgNeCbwDXAA+2nf5KsAL5eVWcmeQJYW1W727bvAecBv9/2+ZM2fmvbZ8sMX28TsAlgbGzsnMnJyZ5yHjhwgMWLF/f9fR7J43te6Wne6mVLDrttrjPOFnPOrlHIOQoZwZy9WrNmzY6qGp8+vqjfB0xyEp2f6k8DXgb+lM5pnjlTVZuBzQDj4+M1MTHR035TU1P0Orfbyuvu6XFmb0/jrvWHz9Bvxvlmztk1CjlHISOYc1CDnCb658DzVfV3VfWPwNeA9wFL22kjgOXAnra8B1gB0LYvAX7YPT7DPpKkeTBIGfwtcH6St7Vz/xcATwH3A5e3ORuAu9ry1rZO2/6t6pyj2gpc0e42Og1YBTw0QC5J0lHq+zRRVT2YZAvwHeAg8CidUzj3AJNJPt3Gbm273Ar8cZKdwD46dxBRVU8muZNOkRwErq6qn/WbS5J09PouA4CqugG4Ydrwc8xwN1BV/QPwm4d5nBvpXIiWJC0AfwNZkmQZSJIsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkBnw7Cs2dXt8+e9dNl85xEkm/CDwykCRZBpIky0CShGUgScIykCTh3UTz6s3uELp29UGuatu9Q0jSfPPIQJJkGUiSLANJEpaBJAnLQJKEZSBJwjKQJGEZSJIYsAySLE2yJclfJ3k6yXuTnJxkW5Jn2+eT2twk+VySnUm+m+TsrsfZ0OY/m2TDoN+UJOnoDHpkcAvwjap6F/Ae4GngOmB7Va0Ctrd1gIuBVe1jE/AFgCQnAzcA5wHnAjccKhBJ0vzouwySLAF+A7gVoKp+WlUvA+uA29u024HL2vI64MvV8QCwNMmpwEXAtqraV1X7gW3A2n5zSZKOXqqqvx2Ts4DNwFN0jgp2ANcAe6pqaZsTYH9VLU1yN3BTVf1l27Yd+AQwARxfVZ9u478HvFZVfzDD19xE56iCsbGxcyYnJ3vKeuDAARYvXnzU3+Pje1456n36NXYCvPRaZ3n1siU9f+3Vy5bMYao36ve5nG/mnD2jkBHM2as1a9bsqKrx6eODvFHdIuBs4Leq6sEkt/DzU0IAVFUl6a9tZlBVm+kUEOPj4zUxMdHTflNTU/Q6t9tVPf7pydlw7eqDfPbxzsuxa/1Ez1971/qJOUz1Rv0+l/PNnLNnFDKCOQc1yDWD3cDuqnqwrW+hUw4vtdM/tM972/Y9wIqu/Ze3scONS5LmSd9lUFU/AF5IcnobuoDOKaOtwKE7gjYAd7XlrcBH2l1F5wOvVNWLwH3AhUlOaheOL2xjkqR5MujfM/gt4CtJ3gI8B3yUTsHcmWQj8H3gQ23uvcAlwE7g1TaXqtqX5FPAw23eJ6tq34C5JElHYaAyqKrHgDdciKBzlDB9bgFXH+ZxbgNuGySLJKl//gayJMkykCRZBpIkLANJEpaBJAnLQJKEZSBJwjKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSsAwkSVgGkiQsA0kSsGihAyyEldfds9ARJGmoeGQgSbIMJEmWgSQJy0CSxCyUQZLjkjya5O62flqSB5PsTPLVJG9p429t6zvb9pVdj3F9G38myUWDZpIkHZ3ZODK4Bni6a/0zwM1V9U5gP7CxjW8E9rfxm9s8kpwBXAG8G1gLfD7JcbOQS5LUo4HKIMly4FLgi209wPuBLW3K7cBlbXldW6dtv6DNXwdMVtVPqup5YCdw7iC5JElHJ1XV/87JFuC/AG8H/gNwFfBA++mfJCuAr1fVmUmeANZW1e627XvAecDvt33+pI3f2vbZMu3LkWQTsAlgbGzsnMnJyZ5yHjhwgMWLF7++/vieV/r5dufU2Anw0mud5dXLlvSccfWyJXOY6o2mP5fDypyzZxQygjl7tWbNmh1VNT59vO9fOkvyAWBvVe1IMjFIuF5V1WZgM8D4+HhNTPT2Zaempuiee9UQ/tLZtasP8tnHOy/HrvUTPWfctX5iDlO90fTncliZc/aMQkYw56AG+Q3k9wEfTHIJcDzwy8AtwNIki6rqILAc2NPm7wFWALuTLAKWAD/sGj+kex9J0jzo+5pBVV1fVcuraiWdC8Dfqqr1wP3A5W3aBuCutry1rdO2f6s656i2Ale0u41OA1YBD/WbS5J09ObivYk+AUwm+TTwKHBrG78V+OMkO4F9dAqEqnoyyZ3AU8BB4Oqq+tkc5JIkHcaslEFVTQFTbfk5ZrgbqKr+AfjNw+x/I3DjbGSRJB09fwNZkmQZSJIsA0kSloEkCctAkoRlIEnCMpAkYRlIkrAMJElYBpIkLANJEpaBJAnLQJLE3LyFtebRyl7/ItpNl85xEkmjzCMDSZJlIEmyDCRJWAaSJLyA/AvDC82S3oxHBpIky0CSZBlIkrAMJElYBpIkLANJEgOUQZIVSe5P8lSSJ5Nc08ZPTrItybPt80ltPEk+l2Rnku8mObvrsTa0+c8m2TD4tyVJOhqDHBkcBK6tqjOA84Grk5wBXAdsr6pVwPa2DnAxsKp9bAK+AJ3yAG4AzgPOBW44VCCSpPnRdxlU1YtV9Z22/H+Bp4FlwDrg9jbtduCytrwO+HJ1PAAsTXIqcBGwrar2VdV+YBuwtt9ckqSjl6oa/EGSlcC3gTOBv62qpW08wP6qWprkbuCmqvrLtm078AlgAji+qj7dxn8PeK2q/mCGr7OJzlEFY2Nj50xOTvaU78CBAyxevPj19cf3vNLX9zmXxk6Al17rLK9etmTBMq5etuRNt09/LoeVOWfPKGQEc/ZqzZo1O6pqfPr4wG9HkWQx8GfAx6vqR51//zuqqpIM3jY/f7zNwGaA8fHxmpiY6Gm/qakpuude1eNbM8yna1cf5LOPd16OXesnFizjrvUTb7p9+nM5rMw5e0YhI5hzUAPdTZTkl+gUwVeq6mtt+KV2+of2eW8b3wOs6Np9eRs73LgkaZ4McjdRgFuBp6vqD7s2bQUO3RG0Abira/wj7a6i84FXqupF4D7gwiQntQvHF7YxSdI8GeQ00fuADwOPJ3msjf1H4CbgziQbge8DH2rb7gUuAXYCrwIfBaiqfUk+BTzc5n2yqvYNkEuSdJT6LoN2ITiH2XzBDPMLuPowj3UbcFu/WSRJg/E3kCVJloEkyTKQJGEZSJKwDCRJWAaSJCwDSRKWgSQJy0CShGUgScIykCRhGUiSmIU/bqNjy8oj/FGda1cf5Krr7mHXTZfOUyJJ88EjA0mSZSBJsgwkSVgGkiQsA0kSloEkCW8tVZ+OdAvqId6CKo0GjwwkSR4ZaG55BCGNBo8MJEmWgSTJMpAkMUTXDJKsBW4BjgO+WFU3LXAkzaNery306trVB5mY1UeUjm1DcWSQ5Djgj4CLgTOAK5OcsbCpJOkXx7AcGZwL7Kyq5wCSTALrgKcWNJVG2mwfbXjHk45lqaqFzkCSy4G1VfWv2/qHgfOq6mPT5m0CNrXV04FnevwSpwB/P0tx58ooZARzzrZRyDkKGcGcvfqnVfWO6YPDcmTQk6raDGw+2v2SPFJV43MQadaMQkYw52wbhZyjkBHMOaihuGYA7AFWdK0vb2OSpHkwLGXwMLAqyWlJ3gJcAWxd4EyS9AtjKE4TVdXBJB8D7qNza+ltVfXkLH6Joz61tABGISOYc7aNQs5RyAjmHMhQXECWJC2sYTlNJElaQJaBJOnYLoMka5M8k2RnkusWOs8hSW5LsjfJE11jJyfZluTZ9vmkhczYMq1Icn+Sp5I8meSaYcua5PgkDyX5q5bxP7fx05I82F77r7YbExZckuOSPJrk7rY+dDmT7EryeJLHkjzSxobmNW95libZkuSvkzyd5L1DmPH09hwe+vhRko8PW85DjtkyGPK3uPgSsHba2HXA9qpaBWxv6wvtIHBtVZ0BnA9c3Z7DYcr6E+D9VfUe4CxgbZLzgc8AN1fVO4H9wMYFzNjtGuDprvVhzbmmqs7quh9+mF5z6LyP2Teq6l3Ae+g8p0OVsaqeac/hWcA5wKvA/2LIcr6uqo7JD+C9wH1d69cD1y90rq48K4EnutafAU5ty6cCzyx0xhky3wX8i2HNCrwN+A5wHp3f8Fw0038LC5hvOZ3/+d8P3A1kSHPuAk6ZNjY0rzmwBHiedgPMMGacIfOFwP8e5pzH7JEBsAx4oWt9dxsbVmNV9WJb/gEwtpBhpkuyEvg14EGGLGs79fIYsBfYBnwPeLmqDrYpw/La/zfgd4D/19Z/heHMWcA3k+xobwEDw/Wanwb8HfA/2ym3LyY5keHKON0VwB1teShzHstlMLKq8yPD0Nzzm2Qx8GfAx6vqR93bhiFrVf2sOofiy+m86eG7FjLPTJJ8ANhbVTsWOksPfr2qzqZzivXqJL/RvXEIXvNFwNnAF6rq14AfM+1UyxBkfF27DvRB4E+nbxumnMdyGYzaW1y8lORUgPZ57wLnASDJL9Epgq9U1dfa8FBmraqXgfvpnG5ZmuTQL1UOw2v/PuCDSXYBk3ROFd3C8OWkqva0z3vpnOM+l+F6zXcDu6vqwba+hU45DFPGbhcD36mql9r6UOY8lstg1N7iYiuwoS1voHN+fkElCXAr8HRV/WHXpqHJmuQdSZa25RPoXNN4mk4pXN6mLfjzWVXXV9XyqlpJ57/Fb1XVeoYsZ5ITk7z90DKdc91PMESveVX9AHghyelt6AI6b3c/NBmnuZKfnyKCYc250Bct5viizSXA39A5h/y7C52nK9cdwIvAP9L5KWcjnfPH24Fngb8ATh6CnL9O5xD2u8Bj7eOSYcoK/CrwaMv4BPCf2vg/Ax4CdtI5PH/rQj+fXZkngLuHMWfL81ft48lD/98M02ve8pwFPNJe9z8HThq2jC3nicAPgSVdY0OXs6p8OwpJ0rF9mkiS1CPLQJJkGUiSLANJEpaBJAnLQJKEZSBJAv4/rU3MUFwJk3AAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QDutsSYyuxIM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "max_seq_len = 25"
      ],
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VLt2E9TAu3D4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# tokenize and encode sequences in the training set\n",
        "tokens_train = tokenizer.batch_encode_plus(\n",
        "    train_text.tolist(),\n",
        "    max_length = max_seq_len,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True,\n",
        "    return_token_type_ids=False\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the validation set\n",
        "tokens_valid = tokenizer.batch_encode_plus(\n",
        "    valid_text.tolist(),\n",
        "    max_length = max_seq_len,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True,\n",
        "    return_token_type_ids=False\n",
        ")\n",
        "\n",
        "# tokenize and encode sequences in the test set\n",
        "tokens_test = tokenizer.batch_encode_plus(\n",
        "    test_text.tolist(),\n",
        "    max_length = max_seq_len,\n",
        "    pad_to_max_length=True,\n",
        "    truncation=True,\n",
        "    return_token_type_ids=False\n",
        ")"
      ],
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lZYxtFmTvGyI",
        "colab_type": "text"
      },
      "source": [
        "Convert Integer Sequences to Tensors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wkp77_-Du-bw",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# for train set\n",
        "train_seq = torch.tensor(tokens_train['input_ids'])\n",
        "train_mask = torch.tensor(tokens_train['attention_mask'])\n",
        "train_y = torch.tensor(train_labels.tolist())\n",
        "\n",
        "# for validation set\n",
        "valid_seq = torch.tensor(tokens_valid['input_ids'])\n",
        "valid_mask = torch.tensor(tokens_valid['attention_mask'])\n",
        "valid_y = torch.tensor(valid_labels.tolist())\n",
        "\n",
        "# for test set\n",
        "test_seq = torch.tensor(tokens_test['input_ids'])\n",
        "test_mask = torch.tensor(tokens_test['attention_mask'])\n",
        "test_y = torch.tensor(test_labels.tolist())"
      ],
      "execution_count": 15,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I3u8NakavdUU",
        "colab_type": "text"
      },
      "source": [
        "Create Dataloaders "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pojOCN-JvbGI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "#define a batch size\n",
        "batch_size = 32\n",
        "\n",
        "# wrap tensors\n",
        "train_data = TensorDataset(train_seq, train_mask, train_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "train_sampler = RandomSampler(train_data)\n",
        "\n",
        "# dataLoader for train set\n",
        "train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "# wrap tensors\n",
        "valid_data = TensorDataset(valid_seq, valid_mask, valid_y)\n",
        "\n",
        "# sampler for sampling the data during training\n",
        "valid_sampler = SequentialSampler(valid_data)\n",
        "\n",
        "# dataLoader for validation set\n",
        "val_dataloader = DataLoader(valid_data, sampler = valid_sampler, batch_size=batch_size)"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5OBuQ4NAv6AQ",
        "colab_type": "text"
      },
      "source": [
        "Freeze BERT parameters"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ha7XjXIZvPTc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# freeze all the parameters\n",
        "for param in bert.parameters():\n",
        "    param.requires_grad = False"
      ],
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6ASrQhanv9M8",
        "colab_type": "text"
      },
      "source": [
        "Define Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HxG_w8Z-v71c",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BERT_Arch(nn.Module):\n",
        "\n",
        "    def __init__(self, bert):\n",
        "      super(BERT_Arch, self).__init__()\n",
        "      self.bert = bert \n",
        "      # dropout layer\n",
        "      self.dropout = nn.Dropout(0.1)\n",
        "      # relu activation function\n",
        "      self.relu =  nn.ReLU()\n",
        "      # dense layer 1\n",
        "      self.fc1 = nn.Linear(768,512)\n",
        "      # dense layer 2 (Output layer)\n",
        "      self.fc2 = nn.Linear(512,30)\n",
        "      #softmax activation function\n",
        "      self.softmax = nn.LogSoftmax(dim=1)\n",
        "\n",
        "    #define the forward pass\n",
        "    def forward(self, sent_id, mask):\n",
        "      #pass the inputs to the model  \n",
        "      _, cls_hs = self.bert(sent_id, attention_mask=mask)\n",
        "      x = self.fc1(cls_hs)\n",
        "      x = self.relu(x)\n",
        "      x = self.dropout(x)\n",
        "      # output layer\n",
        "      x = self.fc2(x)\n",
        "      # apply softmax activation\n",
        "      x = self.softmax(x)\n",
        "      return x"
      ],
      "execution_count": 18,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WDwEZWhRwcZ0",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# pass the pre-trained BERT to our define architecture\n",
        "model = BERT_Arch(bert)\n",
        "model = model.cuda()"
      ],
      "execution_count": 20,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MmMOdBE8wiVg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# optimizer\n",
        "from transformers import AdamW\n",
        "optimizer = AdamW(model.parameters(), lr = 1e-3)"
      ],
      "execution_count": 21,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HrlUpaldxmCU",
        "colab_type": "text"
      },
      "source": [
        "Find Class Weights"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QvD7QzROxlko",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 53
        },
        "outputId": "862c83dd-8fd1-4f0b-fa4e-baf01042cfb2"
      },
      "source": [
        "from sklearn.utils.class_weight import compute_class_weight\n",
        "class_wts = compute_class_weight('balanced', np.unique(train_labels), train_labels)\n",
        "print(class_wts)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "[1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1. 1.\n",
            " 1. 1. 1. 1. 1. 1.]\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hP-tfNx1xqbk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# convert class weights to tensor\n",
        "weights= torch.tensor(class_wts,dtype=torch.float)\n",
        "weights = weights.cuda()\n",
        "\n",
        "# loss function\n",
        "cross_entropy  = nn.NLLLoss(weight=weights) \n",
        "\n",
        "# number of training epochs\n",
        "epochs = 50"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SnVj78cYxuQ4",
        "colab_type": "text"
      },
      "source": [
        "Fine-Tune BERT "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QHhbo_vOxtSY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def train():\n",
        "  model.train()\n",
        "  total_loss, total_accuracy = 0.0, 0.0\n",
        "\n",
        "  # empty list to save model predictions\n",
        "  total_preds=[]\n",
        "\n",
        "  # iterate over batches\n",
        "  for step,batch in enumerate(train_dataloader):\n",
        "    \n",
        "    # progress update after every 50 batches.\n",
        "    if step % 100 == 0 and not step == 0:\n",
        "      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(train_dataloader)))\n",
        "\n",
        "    # push the batch to gpu\n",
        "    batch = [r.cuda() for r in batch]\n",
        "\n",
        "    sent_id, mask, labels = batch\n",
        "\n",
        "    # clear previously calculated gradients \n",
        "    model.zero_grad()        \n",
        "\n",
        "    # get model predictions for the current batch\n",
        "    preds = model(sent_id, mask)\n",
        "\n",
        "    # compute the loss between actual and predicted values\n",
        "    loss = cross_entropy(preds, labels)\n",
        "\n",
        "    # add on to the total loss\n",
        "    total_loss = total_loss + loss.item()\n",
        "\n",
        "    # backward pass to calculate the gradients\n",
        "    loss.backward()\n",
        "\n",
        "    # clip the the gradients to 1.0. It helps in preventing the exploding gradient problem\n",
        "    torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "    # update parameters\n",
        "    optimizer.step()\n",
        "\n",
        "    # model predictions are stored on GPU. So, push it to CPU\n",
        "    preds=preds.detach().cpu().numpy()\n",
        "\n",
        "    # append the model predictions\n",
        "    total_preds.append(preds)\n",
        "\n",
        "  # compute the training loss of the epoch\n",
        "  avg_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "  # predictions are in the form of (no. of batches, size of batch, no. of classes).\n",
        "  # reshape the predictions in form of (number of samples, no. of classes)\n",
        "  total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "  #returns the loss and predictions\n",
        "  return avg_loss, total_preds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zZEMdDdiz1OY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# function for evaluating the model\n",
        "def evaluate():\n",
        "  \n",
        "  print(\"\\nEvaluating...\")\n",
        "  \n",
        "  # deactivate dropout layers\n",
        "  model.eval()\n",
        "\n",
        "  total_loss, total_accuracy = 0.0, 0.0\n",
        "  \n",
        "  # empty list to save the model predictions\n",
        "  total_preds = []\n",
        "\n",
        "  # iterate over batches\n",
        "  for step,batch in enumerate(val_dataloader):\n",
        "    \n",
        "    # Progress update every 50 batches.\n",
        "    if step % 50 == 0 and not step == 0:\n",
        "      \n",
        "      # Calculate elapsed time in minutes.\n",
        "      # elapsed = format_time(time.time() - t0)\n",
        "            \n",
        "      # Report progress.\n",
        "      print('  Batch {:>5,}  of  {:>5,}.'.format(step, len(val_dataloader)))\n",
        "\n",
        "    # push the batch to gpu\n",
        "    batch = [t.cuda() for t in batch]\n",
        "\n",
        "    sent_id, mask, labels = batch\n",
        "\n",
        "    # deactivate autograd\n",
        "    with torch.no_grad():\n",
        "      \n",
        "      # model predictions\n",
        "      preds = model(sent_id, mask)\n",
        "\n",
        "      # compute the validation loss between actual and predicted values\n",
        "      loss = cross_entropy(preds,labels)\n",
        "\n",
        "      total_loss = total_loss + loss.item()\n",
        "\n",
        "      preds = preds.detach().cpu().numpy()\n",
        "\n",
        "      total_preds.append(preds)\n",
        "\n",
        "  # compute the validation loss of the epoch\n",
        "  avg_loss = total_loss / len(val_dataloader) \n",
        "\n",
        "  # reshape the predictions in form of (number of samples, no. of classes)\n",
        "  total_preds  = np.concatenate(total_preds, axis=0)\n",
        "\n",
        "  return avg_loss, total_preds"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "A8Nu0b_H0CPM",
        "colab_type": "text"
      },
      "source": [
        "Start Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VC84anUp0BKc",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "6ba0025f-6b75-495e-9ea2-3b0bf498ed06"
      },
      "source": [
        "# set initial loss to infinite\n",
        "best_valid_loss = float('inf')\n",
        "\n",
        "# empty lists to store training and validation loss of each epoch\n",
        "train_losses=[]\n",
        "valid_losses=[]\n",
        "\n",
        "#for each epoch\n",
        "for epoch in range(epochs):\n",
        "     \n",
        "    print('\\n Epoch {:} / {:}'.format(epoch + 1, epochs))\n",
        "    \n",
        "    #train model\n",
        "    train_loss, _ = train()\n",
        "    \n",
        "    #evaluate model\n",
        "    valid_loss, _ = evaluate()\n",
        "    \n",
        "    #save the best model\n",
        "    if valid_loss < best_valid_loss:\n",
        "        best_valid_loss = valid_loss\n",
        "        torch.save(model.state_dict(), 'saved_weights.pt')\n",
        "    \n",
        "    # append training and validation loss\n",
        "    train_losses.append(train_loss)\n",
        "    valid_losses.append(valid_loss)\n",
        "    \n",
        "    print(f'\\nTraining Loss: {train_loss:.3f}')\n",
        "    print(f'Validation Loss: {valid_loss:.3f}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            " Epoch 1 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 3.016\n",
            "Validation Loss: 2.616\n",
            "\n",
            " Epoch 2 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.586\n",
            "Validation Loss: 2.392\n",
            "\n",
            " Epoch 3 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.432\n",
            "Validation Loss: 2.235\n",
            "\n",
            " Epoch 4 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.337\n",
            "Validation Loss: 2.173\n",
            "\n",
            " Epoch 5 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.291\n",
            "Validation Loss: 2.074\n",
            "\n",
            " Epoch 6 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.251\n",
            "Validation Loss: 2.059\n",
            "\n",
            " Epoch 7 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.217\n",
            "Validation Loss: 2.050\n",
            "\n",
            " Epoch 8 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.198\n",
            "Validation Loss: 2.075\n",
            "\n",
            " Epoch 9 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.175\n",
            "Validation Loss: 1.986\n",
            "\n",
            " Epoch 10 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.156\n",
            "Validation Loss: 1.978\n",
            "\n",
            " Epoch 11 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.142\n",
            "Validation Loss: 1.965\n",
            "\n",
            " Epoch 12 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.131\n",
            "Validation Loss: 1.951\n",
            "\n",
            " Epoch 13 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.117\n",
            "Validation Loss: 1.958\n",
            "\n",
            " Epoch 14 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.117\n",
            "Validation Loss: 1.944\n",
            "\n",
            " Epoch 15 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.106\n",
            "Validation Loss: 1.972\n",
            "\n",
            " Epoch 16 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.102\n",
            "Validation Loss: 1.931\n",
            "\n",
            " Epoch 17 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.086\n",
            "Validation Loss: 1.935\n",
            "\n",
            " Epoch 18 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.087\n",
            "Validation Loss: 1.962\n",
            "\n",
            " Epoch 19 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.078\n",
            "Validation Loss: 1.904\n",
            "\n",
            " Epoch 20 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.071\n",
            "Validation Loss: 1.890\n",
            "\n",
            " Epoch 21 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.072\n",
            "Validation Loss: 1.901\n",
            "\n",
            " Epoch 22 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.061\n",
            "Validation Loss: 1.892\n",
            "\n",
            " Epoch 23 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.058\n",
            "Validation Loss: 1.971\n",
            "\n",
            " Epoch 24 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.051\n",
            "Validation Loss: 1.885\n",
            "\n",
            " Epoch 25 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.052\n",
            "Validation Loss: 1.881\n",
            "\n",
            " Epoch 26 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.050\n",
            "Validation Loss: 1.874\n",
            "\n",
            " Epoch 27 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.044\n",
            "Validation Loss: 1.859\n",
            "\n",
            " Epoch 28 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.045\n",
            "Validation Loss: 1.876\n",
            "\n",
            " Epoch 29 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.044\n",
            "Validation Loss: 1.844\n",
            "\n",
            " Epoch 30 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.034\n",
            "Validation Loss: 1.868\n",
            "\n",
            " Epoch 31 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.034\n",
            "Validation Loss: 1.841\n",
            "\n",
            " Epoch 32 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.030\n",
            "Validation Loss: 1.853\n",
            "\n",
            " Epoch 33 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.027\n",
            "Validation Loss: 1.854\n",
            "\n",
            " Epoch 34 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.019\n",
            "Validation Loss: 1.849\n",
            "\n",
            " Epoch 35 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.024\n",
            "Validation Loss: 1.835\n",
            "\n",
            " Epoch 36 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.018\n",
            "Validation Loss: 1.845\n",
            "\n",
            " Epoch 37 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.021\n",
            "Validation Loss: 1.837\n",
            "\n",
            " Epoch 38 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.013\n",
            "Validation Loss: 1.857\n",
            "\n",
            " Epoch 39 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.019\n",
            "Validation Loss: 1.864\n",
            "\n",
            " Epoch 40 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.002\n",
            "Validation Loss: 1.830\n",
            "\n",
            " Epoch 41 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.017\n",
            "Validation Loss: 1.853\n",
            "\n",
            " Epoch 42 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.013\n",
            "Validation Loss: 1.808\n",
            "\n",
            " Epoch 43 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.008\n",
            "Validation Loss: 1.839\n",
            "\n",
            " Epoch 44 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 2.006\n",
            "Validation Loss: 1.832\n",
            "\n",
            " Epoch 45 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 1.999\n",
            "Validation Loss: 1.807\n",
            "\n",
            " Epoch 46 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 1.993\n",
            "Validation Loss: 1.804\n",
            "\n",
            " Epoch 47 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 1.997\n",
            "Validation Loss: 1.848\n",
            "\n",
            " Epoch 48 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 1.992\n",
            "Validation Loss: 1.832\n",
            "\n",
            " Epoch 49 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 1.999\n",
            "Validation Loss: 1.831\n",
            "\n",
            " Epoch 50 / 50\n",
            "  Batch   100  of  1,443.\n",
            "  Batch   200  of  1,443.\n",
            "  Batch   300  of  1,443.\n",
            "  Batch   400  of  1,443.\n",
            "  Batch   500  of  1,443.\n",
            "  Batch   600  of  1,443.\n",
            "  Batch   700  of  1,443.\n",
            "  Batch   800  of  1,443.\n",
            "  Batch   900  of  1,443.\n",
            "  Batch 1,000  of  1,443.\n",
            "  Batch 1,100  of  1,443.\n",
            "  Batch 1,200  of  1,443.\n",
            "  Batch 1,300  of  1,443.\n",
            "  Batch 1,400  of  1,443.\n",
            "\n",
            "Evaluating...\n",
            "  Batch    50  of    161.\n",
            "  Batch   100  of    161.\n",
            "  Batch   150  of    161.\n",
            "\n",
            "Training Loss: 1.996\n",
            "Validation Loss: 1.818\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "R6BE-_620uFw",
        "colab_type": "text"
      },
      "source": [
        "# Testing"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KVP7q3IX0UpU",
        "colab_type": "text"
      },
      "source": [
        "Load Best Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4zANeZZA0R4w",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "d8cb4f43-9f12-40bd-abf4-a3a3965f0573"
      },
      "source": [
        "#load weights of best model\n",
        "path = 'saved_weights.pt'\n",
        "model.load_state_dict(torch.load(path))"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "e1XQfM_S0Xy4",
        "colab_type": "text"
      },
      "source": [
        "Run for Test Set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2KspYCw3tgqg",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from torch import topk\n",
        "\n",
        "def accuracy(output, target, topk=(1,)):\n",
        "    \"\"\"Computes the precision@k for the specified values of k\"\"\"\n",
        "    maxk = max(topk)\n",
        "    batch_size = target.size(0)\n",
        "\n",
        "    _, pred = output.topk(maxk, 1, True, True)\n",
        "    pred = pred.t()\n",
        "    correct = pred.eq(target.view(1, -1).expand_as(pred))\n",
        "\n",
        "    res = []\n",
        "    for k in topk:\n",
        "        correct_k = correct[:k].view(-1).float().sum(0)\n",
        "        res.append(correct_k.mul_(100.0 / batch_size))\n",
        "    return res"
      ],
      "execution_count": 41,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OztlwK290ah4",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# get predictions for test data\n",
        "with torch.no_grad():\n",
        "  preds = model(test_seq.cuda(), test_mask.cuda())\n",
        "  pred_data = preds.detach().cpu()\n",
        "  preds = preds.detach().cpu().numpy()"
      ],
      "execution_count": 47,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "95hcXEUN0dF8",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 689
        },
        "outputId": "48cc5d1b-4175-492a-f9e2-850434bfdf03"
      },
      "source": [
        "# model's performance\n",
        "preds = np.argmax(preds, axis = 1)\n",
        "print(classification_report(test_y, preds))"
      ],
      "execution_count": 57,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.40      0.25      0.31       190\n",
            "           1       0.28      0.35      0.31       190\n",
            "           2       0.41      0.47      0.44       190\n",
            "           3       0.82      0.88      0.85       190\n",
            "           4       0.27      0.23      0.25       190\n",
            "           5       0.59      0.46      0.52       190\n",
            "           6       0.52      0.59      0.56       190\n",
            "           7       0.61      0.87      0.72       190\n",
            "           8       0.39      0.48      0.43       190\n",
            "           9       0.41      0.44      0.43       190\n",
            "          10       0.63      0.30      0.41       190\n",
            "          11       0.46      0.34      0.39       190\n",
            "          12       0.46      0.51      0.48       190\n",
            "          13       0.26      0.24      0.25       190\n",
            "          14       0.53      0.58      0.55       190\n",
            "          15       0.33      0.29      0.31       190\n",
            "          16       0.54      0.47      0.50       190\n",
            "          17       0.46      0.45      0.46       190\n",
            "          18       0.42      0.47      0.45       190\n",
            "          19       0.30      0.24      0.27       190\n",
            "          20       0.48      0.37      0.42       190\n",
            "          21       0.38      0.57      0.46       190\n",
            "          22       0.35      0.42      0.38       190\n",
            "          23       0.39      0.59      0.47       190\n",
            "          24       0.48      0.44      0.46       190\n",
            "          25       0.39      0.30      0.34       190\n",
            "          26       0.39      0.49      0.43       190\n",
            "          27       0.38      0.09      0.15       190\n",
            "          28       0.68      0.68      0.68       190\n",
            "          29       0.49      0.61      0.54       190\n",
            "\n",
            "    accuracy                           0.45      5700\n",
            "   macro avg       0.45      0.45      0.44      5700\n",
            "weighted avg       0.45      0.45      0.44      5700\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qD0F0SdItsbo",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 72
        },
        "outputId": "b6af7e86-0795-435d-c3aa-6525f60109fe"
      },
      "source": [
        "top_acc = []\n",
        "top_acc = accuracy(pred_data, test_y, topk = (1, 3, 5, 30))\n",
        "print(f\"Top 1 Accuracy is: {top_acc[0]}\")\n",
        "print(f\"Top 3 Accuracy is: {top_acc[1]}\")\n",
        "print(f\"Top 5 Accuracy is: {top_acc[2]}\")"
      ],
      "execution_count": 58,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Top 1 Accuracy is: 44.98245620727539\n",
            "Top 3 Accuracy is: 68.19298553466797\n",
            "Top 5 Accuracy is: 78.49122619628906\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hx0X8yrV8N60",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 319
        },
        "outputId": "81007947-3eef-4bca-bde1-424f0d468154"
      },
      "source": [
        "from sklearn.metrics import confusion_matrix\n",
        "import matplotlib.pyplot as plt \n",
        "cm = confusion_matrix(test_y, preds)\n",
        "plt.clf()\n",
        "plt.imshow(cm, interpolation='nearest', cmap=plt.cm.Wistia)\n",
        "classNames = [str(i) for i in range(30)]\n",
        "plt.title('Confusion Matrix - Test Data')\n",
        "plt.ylabel('True label')\n",
        "plt.xlabel('Predicted label')\n",
        "tick_marks = np.arange(len(classNames))\n",
        "plt.xticks(tick_marks, classNames, rotation=45)\n",
        "plt.yticks(tick_marks, classNames)\n",
        "plt.figure(figsize = (50, 50))\n",
        "plt.show()"
      ],
      "execution_count": 59,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAQ4AAAEcCAYAAAA2r1o1AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO2deZgdVdG435otewIhCSQTIAgkGJBN9n2XTREFFGVXEBA3EFTkQ1T0ByqC+AEugCBgWBQQFAREEfETEJB9U5EtCSQgARKyzFK/P865TM/NPdX33uRO38F6n2eeud3VZ+nu09WnT52qI6qK4zhOLbQUXQHHcQYfrjgcx6kZVxyO49SMKw7HcWrGFYfjODXjisNxnJpxxdFgRGSYiNwoIq+LyDXLkM/HReTW5Vm3IhCRm0Xk0KLr4SwbrjgiIvIxEblPROaLyOzYwLdZDlnvB6wMrKSq+9ebiapeoaq7LYf69ENEdhARFZHryvZvEPffUWU+p4nI5XnHqeoeqnppndVNlb1tvG/zRWRBrPf8zN9qdeSpIrKWIT9MRHoyZfxbRH4mIlNrKOMSETm91ro1A644ABE5HjgH+DbhIV8NOB/YZzlkvzrwtKp2L4e8GsVcYEsRWSmz71Dg6eVVgAQa0t5U9c+qOlJVRwLrxt0rlPap6vONKBf4ayxzDLALsBC4X0TWa1B5zYOq/lf/EW76fGB/45ghBMUyK/6dAwyJsh2AF4ETgDnAbODwKPs6sAToimV8AjgNuDyT9xRAgba4fRjwDPAm8G/g45n9d2XSbQX8DXg9/t8qI7sD+Cbwl5jPrcC4xLmV6v8j4NNxXyswEzgVuCNz7A+AF4A3gPuBbeP+3cvO86FMPb4V67EQWCvu+2SUXwD8KpP/mcDtgCzD/Sy/nmOAi+J9mQmcDrRG2VrAn+I1fAW4Ku6/M+axIJ7PRyqU0+9+ZPb/BvhlZvsa4KVYxp3AunH/UfF6LYll3Bj3fxn4V7xvjwP7Fv2MVLzORVeg6L/Y6LtLDS1xzDeAu4EJwHjg/4BvRtkOMf03gHZgT+AtYMUoP43+iqJ8++2GDoyID+W0KJuYaWhvN1RgLPAacHBMd2DcXinK74iNbyowLG6fkTi3HQiKYyvgnrhvT+AW4JP0VxwHASvFMk+ID8TQSueVqcfzhF5AW7w+d9CnOIYTejWHAdsSHt7Jy3g/376ecfs64Mfx2k4A7gU+FWUzgK8Set5DgW0y+SiwllHO2/ejbP8RwMtl26Poe/k8mJFdApxeln5/YFKs00cIymti0c9J+Z9/qoQH4RW1PyU+DnxDVeeo6lxCT+LgjLwryrtU9SbCG2RanfXpBdYTkWGqOltVH6twzF7AP1T1MlXtVtUZwJPA+zPH/ExVn1bVhcDVwIZWoar6f8BYEZkGHAL8vMIxl6vqq7HMswgPQ955XqKqj8U0XWX5vUW4jt8HLgc+o6ov5uRXNSKyMkEJfl5VF6jqHOBs4KPxkC7Cp+QkVV2kqncth2JnERQ7AKp6saq+qaqLCcp1AxEZk0qsqteo6ixV7VXVq4B/AJsth3otV1xxwKvAOBFpM46ZBDyX2X4u7ns7jzLF8xYwstaKqOoCwlvmaGC2iPxWRNapoj6lOnVmtl+qoz6XAccBOxLe1P0QkS+KyBPRQjSP8BkwLifPFyyhqt5D+DQTgoKriIg8lhmI3DanzBKrE3o5s0VkXqzzjwk9D4CTYrn3xvyPqDJfi07gP7HOrSJyhoj8S0TeAJ6NxySvmYgcIiIPZuq7nnV8UbjigL8Ci4EPGsfMIjTCEqvFffWwgNBFL7FKVqiqt6jqroTPlCeBn1ZRn1KdZtZZpxKXAccCN8XewNvEh/Uk4ADCZ9gKhO92KVU9kafpfi0inyb0XGbF/Ctnorqu9g12/rmakyEorcWE8Z0V4t9oVV035vmSqh6pqpOATwHnW5aUKtkXKNXvY4QB9l0ISnZK3F/xmonI6oT7fRzhs3MF4NHM8U3Df73iUNXXCYOA54nIB0VkuIi0i8geIvKdeNgM4BQRGS8i4+LxuabHBA8C24nIarHL+pWSQERWFpF9RGQEocHPJ3y6lHMTMDWakNtE5CPAdMLAXN2o6r+B7Qnf/eWMIozlzAXaRORUYHRG/jIwpRbLSTRdnk4YOzkYOElEzE+qWlDV2YSB4bNEZLSItIjImiKyfSx/fxGZHA9/jfAgl673y8C7qjyPVhFZQ0R+SBgz+noUjSLcx1cJL4tvlyUtL2NErMPcmO/hhB5H0/FfrzgA4vf68cAphJv2AkHrXx8POR24D3gYeAR4IO6rp6zbgKtiXvfT/2FvifWYRejubg8cUyGPV4G9CQOUrxLe1Hur6iv11Kks77tUtVJv6hbgd4TBzOeARfT/DClNbntVRB7IKyd+Gl4OnKmqD6nqP4CTgctEZMiynEMZhwAdBAvFa8AvCb05gE2Be0RkPnAD8DlVfSbKTgMujZ8MByTy3jKmfYMw6Dsa2FRVH4nynxOu1cxY/t1l6S8CpscyrlfVx4GzCL3gl4H3ECxSTYfEkVzHcZyq8R6H4zg144rDcZyaccXhOE7NuOJwHKdmrElPTcO4sS06ZXJrRVlXx9iK+/uo1wRuDBovy4CypOvT1v1mUtbdmjN/y7SCGtdAe4xUlSzBmaTSbknrq09u2nrJydO6p8a1FeP6qVRus5nUhsy69vW/76X/5N23eeG5+bz6yqKqH5ZCFIeI7E5wmGoFLlTVM6zjp0xu5b7fVFYQszsPNMtSc0Ko1SCWpFMlLv7bZRr5WvUZ+/odSdlro20P/96WYekyW9IPeGv3G0lZW09aBrC4Y1JSlmqgoT62tVV6retb3wNl1QegxbjfPS0jkjLrGnW3jk7KwFYsLb0LkzLrXtsvEOhYMrvi/l23vMFMV86Af6qISCtwHrAHYdLSgSIyfaDr4ThO/RQxxrEZ8E9VfUZVlwBXsnziXjiOM0AUoTg66T/j8EX6O2cBICJHxYhc9839j/2t7TjOwNK0VhVV/YmqbqKqm4wf27TVdJz/Sop4ImcCq2a2J7PsXp2O4wwgRVhV/gasLSJrEBTGRwnux0m6OlZi1uSDKsomPXSBWdisDZbyEXsby9pAb/rzqLs1PcqeR0vPgqTs1RV2TcryLBw9LekRfMtK0dGd9ovrlQ6zzNae+WmZYRVY0rGyma9lUbCsUqKLk7KeHHN2a/dbSZllkelqW9HM18LK17rfS5ah/aVN6LVNWxjwHkcMePNPQmSjN4GrE1GuHMdpUooaPDidYF35h6p+q6A6OI5TJ4UoDlW9kxhezXGcwUfTmiuy5thX56a/Px3HGXiaVnFkzbErjR+en8BxnAGjaRWH4zjNiysOx3FqZsDncYjIqoTAv2MJ0bLnASeo6kV2yso6bvb6R5qpJt12VlI2830nJ2V5HpwWI956MilbMCy9JrE1fyHP0xJNzzuxvDC7W5NrA+W68g9dkp63t3BoOkC47f0KPW3GuRrnac0ryaO7dVS6SKMtDF30bFK2aEjNa12/TVdbXriIyrT02OOBqfudHwKgrJyajl4+dAN7qGo7ISr0HEJUZ8dxBglFTACbraoPxN9vAk9QwcnNcZzmpdAxDhGZAmwE3FNBljHHprvwjuMMPIUpDhEZCfyKsCDwUhPz+5tjjYhHjuMMOIUoDhFpJyiNK1T12iLq4DhO/RQROlAIS989oarfH+jyHcdZdopwq9+asMDwIyLyYNx3sqrelE4iSRd4xYq0bZtcVzmtfA3gPl46LZ0ujwXD10nK8sxlKXLNZVaQWsOE2dW+Ul3pwDa5WuSeS065Keo140JexPY0i4ZOaUyZdd7P3pahZplWVPZaKOJT5T5CTA4F2oFf20rDcZxmo4gex2JgJ1WdH8c67hKRm1W1fCVvx3GalAFXHKqqQGmKX3v8a8QKPI7jNIiirCqtcXxjDnCbqubM43C3esdpJooK5NOjqhsSAhVvJiLrVTjG3eodp0kpdOaoqs4D/gjsXmQ9HMepjSK8Y8cDXao6T0SGAbsCZ9ab39DFL5jyRR1pN5g5//P5pKzzpZ8lZXPH7mmWuaR9fFJmRf/uMdYEtTxnwY7+bXl3WlHXO7rt6I7WtTXXjs0xfbZ3v5aUmV7ChqmxtTd9nmBHdLfWlbUjwdtDd9YasMMX/iMpsyKrW16+YLS/Gk3gRfQ4JgJ/FJGHCXFHJ6jqbwqoh+M4dVKEVeVhYCMROR7YhOBa7zjOIKIoq8pkYC/gwiLKdxxn2ShqcPQc4CQg+WHl5ljHaV6KcHLbG5ijqvdbx7k51nGalyJ6HFsDHxCRZ4ErgZ1E5PIC6uE4Tp0UMTj6FeArACKyA/BFVa28ovTbiXpo7a68CO/i9gl2gYaXYW9ruidjmVzH/8mOqzxr5xOSMssbta17XlLW02IvNFxrsNm+dGlzYq9h4s3D8sLMCwTdW6enqrWIeA95CzWnzZFtXen7srhjYlJmBXMGWDhkiiFLBzqu26sW6En1FXLSlVOEkxuxt/EmMAwYV0QdHMepn0IUR2RHVX2lwPIdx6kTX5DJcZyaKUpxKHCriNwvIkdVOqCfOfYVj3LuOM1EUZ8q26jqTBGZANwmIk+q6p3ZA1T1J8BPADZ47wSP1+E4TURRbvUz4/85wHXAZkXUw3Gc+ihiAtgIERlV+g3sBjw60PVwHKd+ivhUWRn4dVx8ehjwevxLI612BGuDkQvSOuktI0q3tejvzF3tCOidV6Wjp8/a/8SkLHdhaYOW3kVJmTVfxZz7kDN3xFrk2bpfy7TotIXpGl5/xHFr8eiOrpfT6YywA3nUu+h5e5dtqOxqT81+kJrKKWLt2GeABwgTv4YQooA9MdD1cBynfooI5DMG2A44DEBVlwDpSCmO4zQdRQyOrgHMBX4mIn8XkQvjWEc/3DvWcZqXIhRHG7AxcIGqbgQsAL5cfpB7xzpO81KE4ngReDGzJMIvCYrEcZxBQhGDoy8BL4jItLhrZ+Dxga6H4zj1U9TM0c8AV4hIB/AMcHjdOeVEZ54/YqklW5ZLvhYzP1LfQtdW1HXLpJontxa6ttL1tI40yzQxrp9lAl6WfC3XcJX6zJsA0rs4KetqSzt3L8t51nvP0ubWyDK06yxFTACbBlxCUFq9wE7AoQNdD8dx6qeIQD5PARtCWAoSmEmYdu44ziChaLf6nYF/qepzBdfDcZwaKFpxfBSYUUng8zgcp3kpTHHEgdEPANdUkvs8DsdpXorscewBPKCqaS8hx3GakiJjjh5I4jOlHNFuOpZU1i9LOla20xqmNLvMdJTu3pahdaed95WDk7KJt52TlM3aLR05PWCZIg2zoGGesxZ/Bjtiu5WvZWoE29xoeeT2Gh6l1j0Jaa17mr627cbC3Nbi43lYi5PnmebNfFPXL+f6lFPUEpBfAvYDThSRGSJiP4mO4zQVRczj6ASOBkap6nSglTBI6jjOIKGoMY42YJiItAHDgVkF1cNxnDoowldlJvA94HlgNvC6qt5afpxHOXec5qWIT5UVgX0IcTkmASNEZKklIPuZY8cNG+hqOo5jUMSnyi7Av1V1rqp2AdcCWxVQD8dx6qQIc+zzwBYiMhxYSJh2fp+VQKUtadpKLUZdwvLwNE2urfV7U6rhpWkFsLVMrpPuOMssc+ZOaY9cKziwFeS4q21Fs0zL1C3anZT1ttpBkC1zbYumz0XVas720jydL1+alM2a8LGkrKtthXSmOQs5W+fZvSyeyQa9LYnee7MvOq2q94jIC8BrhLv5CFBxNTfHcZqTIsY41iNENl8RGElYGmHVga6H4zj1U8QYx7uBe1T1LVXtBv4EfKiAejiOUydFKI5HgW1FZKU4zrEnFXoc7h3rOM1LEfM4ngDOBG4Ffgc8CCw1SunesY7TvBS16PRFqvpeVd2OMEj6dBH1cBynPgrxjhWRCao6R0RWI4xvbFFEPRzHqY+GKQ4RuRjYG5ijquvFfWOBq4BtRKQHeBb4tKrOs3NTxLDfW1iLAlsRqq25DyqtZpkjFj6VlC0Y/u6kbMiSOUmZNU8DoPOp89Nppx5tpEy7v+dd87rnauRE2rbcxu1o7gvSRYrd1GdNWGryclWs+MadSdl/VtjZTGudy9BFzyZli4ZOScqsaxBILS5tz3NZqpyajq6NS4Ddy/Z9GbhdVYcBpwO/VdXbG1gHx3EaQMMUh6reCZRHOdkHKE3RuxT4YKPKdxyncQz04OjKqjo7/n4JSIbvcnOs4zQvhcUcVVXF+LByc6zjNC8DrTheFpGJAPF/ejTQcZymZaAVxw30Lfd4KPDrAS7fcZzlQCPNsTOAHYBxIvIi8DXgDOBqEfkE8BxwQJW5JSN197bYus9cONlwJbZNe3ZgIcvkOmzRM0nZwiFTkrK8aO0zpx2blHU+dG463frHpcvMMcdaJlfLnD1m/t/MfOeNTodnsa7f4vZVkjIz0jt29HSrDc0fPt3M12L4wn8kZd2to+vKMy9kQfq+pMy0lWnkBLCFhEDET2XmcewPrAKsCRyoqunY8o7jNC0DPY/jUcJM0fSsGcdxmp6G9ThU9U4RmVK27wkAkdq6RY7jNBdFLzqdxOdxOE7z0rSKw+dxOE7z0rSKw3Gc5iU5xiEiP8Se2fnZhtSoIr20JBbhHbb4eTPlgmHT6ipRpSMtrDEidJZFHenwqnkLI1tY5lrL5Nr52zPS6fa2PXKtCPPJaNrY5tY8Fg59V1JmmdAtcytAW0/6XHra0qbR7tYxSZllkgZ4a9jaSZnp5ZrjXWyR8ha3PJ0rYQ2OmksW5CEi/wKmAC2ZeRzvB/YimGnvEpG/qKrte+w4TtORVByq2m+hCREZrqq1jFIeDswHfp6Zx/ECsJ+qdovImfVU2HGc4sntc4vIliLyOPBk3N5ARNJRYyKV3OpV9dYY2RzgbsIyCY7jDDKq+Vg/B3gf8CqAqj4EbLccyj4CuDkl7G+O9UWnHaeZqGqUT1VfKNtV/ygeICJfBbqBK4wyM+ZYX3TacZqJamaOviAiWwEqIu3A54An6i1QRA4jxCLdOcbkcBxnkFGN4jga+AHQCcwCbgE+XU9hIrI7cBKwfS0DraK9tCbMsXlBaE0zm2EybOt5Mynr7TVMtZCsK5A0KwNgBEG2FqvOw1pYetaeJyZlnS9eZOY7c/InkjLTVGvmal+j3pb0YuDmIuLGvQZYYsgtU3dvy9CkLM8E3NOS9titd3FtNa4PwJL2ykH38p6jcnKPVtVXgI/XlCtJc+x3CevFPi8iXcD1qnpIrXk7jlMs1VhV3iUiN4rIXBGZIyK/FpH0LJw+Dgc2BR5T1cmqehEwRVU7YpTzUwB3QnGcQUg1g6O/AK4GJgKTgGuAGXmJEubYbP91BLUu5uA4TlNQjeIYrqqXqWp3/LscSH/Y5SAi34oTwT4OnGoc12eOfSX9je44zsCTVBwiMjauvHaziHxZRKaIyOoichJwU70FqupXVXVVgik26UTRzxw7rm495ThOA7AGR+8nfEqUou58KiNT4CvLWPYVBAX0tWXMx3GcAcbyVVljeRcmImurailC6z7EaeyO4wwuqjLeish6wHQyYxuq+vOcNJXMsXuKyDRgLGGw9T3VlK/SRlfb2IqynhY7yE97dzoecnfbCklZV4493EJ70vMxejuSi9eZbtjWXAywFzC2Inxbi3Jb8zQAOq/+djrtAWmX/LyI7fXO1WjRJUlZjxjR7rGvr3X9rHSWO37IOD2jxWqbFnmLTqfnsyznKOci8jXCMgfTCZ8WewB3AabioIJ3LHCRiKwKXAh0EZaBdBxnkFGNVWU/YGfgJVU9HNgASEcviSQWnQY4mzB71E2xjjNIqUZxLFTVXqBbREYTlm1Mh7EyEJF9gJnRwzbvWA9W7DhNSjVjHPeJyArATwmWlvnAX2stSESGAycDu1VzvKr+BPgJwAabTPTeieM0EdX4qpTWFvyRiPwOGK2qD9dR1prAGsBDcV2VycADIrKZqvpYh+MMIqxgxRtbMlV9oJaCVPURYEImj2eBTaITneM4gwirx3GWIVNgJyvjSotOR0e3OuhNmr1ae+3xj5QbMdjmT8sdP29BYMuFOy/ydQrLVR9yzLGG+3ZX24rpMg3XeLBNrp1PX5BON/UYM9+RC9Id2reGrmmkNEyKOZHprevb0ZWOpL9g2NR0njnXz24n6fpYEfjN6PwY7brGCPvWBLAda8ppaSotOn0acCQwF5gHbMYyTF93HKcYBnrRaYCzVXXD+OdKw3EGIQ1THMY8DsdxBjlFLAF5nIg8LCIXi0jyA9ujnDtO81JNBDARkYNE5NS4vZqIbFZneRcQzLIbArMxBmA9yrnjNC/V9DjOB7YEDozbbwLn1VOYqr6sqj1xJupPCYOjjuMMMqqZObq5qm4sIn8HUNXXRHJsPglEZKKqzo6b+wKPVpeyJWluVCMyONgRsy3vxa70V1Suaa+969WkrLvV8tJM59vVvpJZpoVl3pw/Yv10wpzFjdu65yVllsm189a0Vy3AzN3SZl6rzF7DizXvXCwTu2WyHjP/b0nZ66M2N8s0o6e3jjDTpsgzASe9bnOeo3KqURxdItJKdEoTkfHkR7hPudXvICI7E9zqlwCX1VRbx3GagmoUx7nAdcAEEfkWwVv2lCrSVVp0+hlCHI69VHWxiEywMnAcpzmpxlflChG5n+BaL8AHVTV3JTdVvVNEppTtPgY4Q1UXx2Pm1Fxjx3EKpxqrymqE9U9uBG4AFsR99TAV2FZE7hGRP4nIpka57lbvOE1KNZ8qv6UvaPFQgofrU8C6dZY3FtiCsFjT1SLyrkpryLpbveM0L9V8qvSLCxq9Zo9NHJ7Hi8C1UVHcKyK9wDiC74rjOIOEmmeORnd6286U5npgRwARmQp0AO5W7ziDjGqCFR+f2WwBNiasWp+XrpI5dmdgZxE5Meb1QqXPlKXRpDt6R5fdWbHc6i3sVe5tG7s558KYT2BG8O6xx3kst/oFw95dV76i3XaZxnQea+6INU8DoPP3RvT0XYy01lyNnLk3llv50MVpt/rXRyaH6ehYko4gD9BVZyRzKzRDXmT19NyR2kYDqhnjGJX53U0Y8/hVFekqRjkvCUXkLOD1KuvpOE4TYSqOOPFrlKp+sdaME+bYUr4CHEBOMCDHcZoTa+3YNlXtAbZuQLnbAi9nVnWrVL57xzpOk2L1OO4ljGc8KCI3ANcAby8TparXLkO5BwIzrAP6m2NXcXOs4zQR1YxxDAVeJXxWlOZzKFCX4hCRNuBDwHvrSe84TvFYimNCtKg8Sv9V62HZVmHbBXhSVV9chjwcxykQS3G0AiOpHDo6V3EkzLH3EzxiF4nIfcCxqnpvfjUlGak7L+K4GAsRqxqLQxsu2lbU8Dxae+an8zVcmy3zMMASwxw7pCu9bM2iIelF+fJMwFYzmD98vaQsb9HpmTt/OSnrvCVtqp29y+eSMiuiOIBo2sS5uGOikTBt5l3SPt4sc8TCp5MyK3q62f5ywgekF9BefotOz1bVb9SUW38qecfeChyiqjeLyJ7AdwhLKDiOM4iwZsXUpoLKSAQrVqDURRhDFRPJHMdpPqwex84NKO/zwC0i8j2C0toqdaCIHAUcBdC5mv054jjOwJLscahqI5Y2OAb4gqquCnyBzEzSCuVnghWnv98dxxl4Bnp5hEPpM+NegwcrdpxByUArjlnA9vH3TkBy5qjjOM1LNRPA6qLSotOEdWN/ECeBLSKOYeTTmzQN5i3GbJrZhlhervWPDZvRqw2zoGVm686rj2GGs0yuVkT2npahZpHWtW8xrrvlVQugLUOSsrk7Hp6UTZx9ZVL28iofNssctuiZpGzBsHWSsvYls5OyJZYZF1jUMSkps9qtNRnC8rAG24u6FhqmOKi86PQGhOjmHcAcvMfhOIOSgV50+kLgyzGq2HXAiQ0s33GcBjHQi05PBe6Mv28D7P6j4zhNyUAPjj4G7BN/7w8kP77drd5xmpeBVhxHAMfGdVpGEcY7KuKLTjtO89LIwdGlUNUngd3g7WDFew1k+Y7jLB8GVHGIyARVnSMiLYRlJH+0rHnmeSC2d7+Wro8R9DXtRUiuB6LVkbPMbNKT9uTNxfDStMzD9iLYNj0tadNer2HKNU2NOXS3jknK5k7YOylb+e/JScoAvLzRJ5Iy61zoeTMpstpXyDdtdrZM3cmFo4GOxTnRKrorTwiv9Z40ch7HrwlWlQ4R6QJ+CdwtIp8hrB+7GJgoItepavrpdhyn6WjkGMfRwJaqKoTV295LsKRcC3xTVccCtwPp4AuO4zQljTTHzo6LN6GqbwJPAJ0Eq8ql8bBLgQ82qg6O4zSGAbGqxGUSNgLuAVZW1dI83ZeAiismuTnWcZqXhisOERlJWMDp86raL/5dXMWt4sx7N8c6TvPSUMUhIu0EpXFFZjmFl0VkYpRPJPisOI4ziGiY4oirtV0EPKGq38+IbiDE5SD+/3Wj6uA4TmNo5DyOfYGDgcUi8inC2ixHAY8DPxSR/0dY9GmP/KwkOa8iz43YioJuRRVvMd3189zCjTkgxhQQqz4j33rcLHP+iHRUcWtOSnvCrg/Q1TbWLNN671jXr7fVXrQ7f55MIl9jXsTMTb5gpu381RlJ2ex909HTu1tHJWXLgjVXw2Kx4aoPJOf7mPOWKtDIT5W/Au9V1aHABOAt4FngboJp9k/ApxsUotBxnAbSsB5HtJzMjr/fFJEngE5VvQ0gfMk4jjMYKcIcW20aN8c6TpNSqDnWws2xjtO8FGGOdRxnkFOEOdZxnEFOEebYowhxOFqBu0TkL6qas2pcetHp9q5XzJRd7eNqrXcubT2v22VKnhmzdhYNmWwfYJkwDZd7y029pXeRWaQVMXvIoplJ2aIcc+yY+X9Lyt4YsWFS1t1anwkTYPa+n0/KJv7l3KRs5nZpH83hC+1Y3AuHrJYWGvfTcoG3IsSD5eqfu458P4owx54PDFPVFuAHwH0NrIPjOA1gwL1jVfVWVe2Oh90N5LxKHcdpNoo2xx4B3JxIkzHHVl6MyXGcYijMHCsiXwW6gSsqpfNFpx2neWlozNGUOVZEDgP2BnaOrvWO4wwiGhlztKI5VkR2B04CtldV/2++yD8AABcOSURBVAZxnEFII3scWxPMsY+IyINx38nAucAQ4Lbor3K3qh6dm1vCPJW3gHFqsWrIiV5tkGfitcq0vBAt82eet6R5nobZ1PLINb18gbbueUnZIsPUaNUV4I0RG9dVp5aeBel0Yjd1677M2uaEpKzz6m8nZTP3rz+cbkfX3KRsSUfFoHlV0dZTefJ2npd5OY0c43gOuIOgnNqBn6nqTcAMoBTVfA7wjQbWwXGcBtBIxdENnKCq04EtgE+LyHTgu6q6vqpuCPwGOLWBdXAcpwEUMY8j21caQa1T1hzHKZwBWcmtfB6HiHwLOAR4HdgxkaY0PZ3O1dJRvBzHGXgKmcehql9V1VUJcziOq5TO53E4TvNStFv9FcCHG1kHx3GWP0XM41hbVUtug/sAT1aXYX06zjI3Wnm2GqbRtp75ZpnWQs6m+dOqaw71prWCCueZMM3Fj5e8nJTlmRMtj+eulrQpXHNM8/Wywpt/TcpmHnByUtZ5e9pUCzBz53RaczHwOoM5h3wrf/bX2n6KcKv/hIhMI6wnOxF4TwPr4DhOAxhwt3pV/TBhSYRHgOcJy0A6jjOIKGLRaYCzCdPO3RTrOIOQAXerF5F9gJmq+lBOGnerd5wmZUDNsYTZpCdTxWxRN8c6TvMy0ObYNYE1gIdE5FlC9K8HRGSVRtbDcZzly4CaY1X1EcJAaemYZ4FNVNWOOOw4TlNRhDl2M+BIYC4wCdgFuNLOSpHexXVVYsiS2UnZ4o7OpKyrfaW6ygPb3dxedDo9D6Gt+7WkDOz6mvMijBABudfcmE+wpH18UmbN8QB7nseK8+5Iyl4ftWlSljfHY8iSdFT2eaM2T8qsEAGzdjzRLLPzqfOTstlrHZZOWOecJgBNpq1tSdYiopwDnK2qG6pqh6rmKA3HcZqNAV90ulHlOY4zcBQV5fw4EXlYRC4WkRUTadwc6zhNShHesRcQrCsbEnokZ1VK5+ZYx2leBtw7VlVfVtUeVe0FfkoYLHUcZxAx4ItOi8jEzGH7Ao82qg6O4zSGIsyxB4rIzgTv2CXAZdVlV1nHpaI2l1jSPiEps1yJTRNmm72odMp1GTBNaVaZ1uLQYEf47m4dZaZN5pljju1pTUcGt6JmL0uU7nmjt64rXZ7buBWV3cK6RnllWibXibeek5TNel/azJtX5vBF/6q4P+9eL3V8TUfXRsocezHwODBGVUcDX29gHRzHaQBFmGOPBM5Q1cVRNqdRdXAcpzEUYY6dCmwrIveIyJ9EpOJ0v/7m2HSUKsdxBp4izLFthPGNLYATgavjQGo/+ptjhzW6mo7j1EARwYpfBK7VwL0E7w17TUXHcZqKATfHAtcT11IRkalAB+DesY4ziChi0emLgYtF5FGCOfZQVa07hGBvi/0ZYy24XC+tOVHOe4wI1bbZNG1yFV1ilmldB9GupKy12zZnW7QbHrtWfXNNy0Zay8zbU+c1CPLupGzEwqeSsvnDpydlbT1v5pSZPpdXdjokKRu+6J9J2cIhq5tlLkyYnXtbaosQX8Si05cB6xGigY0Fvp/KwHGc5qSRPY7SotMPiMgo4H4RuU1VP1I6QETOIiwD6TjOIKKIeRyPw9tjIAcAOzWqDo7jNIai3OoBtgVezqzqVp7G53E4TpNSyKLTkQOBGal0Po/DcZqXRo5xJBedFpE24EPAextZvuM4jWHAF52O7AI8qaovVpkb2lLZE7OrxQ4qbAWTtTwJLZNhngeiZabsaqsY8AywTYZ5wXYt055K2ou1p21IOs8cj0nVdPPpbU3fFzOYM/Zi1vUuuGzdE7CDPb8xMv1+s657V/sIu1LGuVhtYfGQyUnZpAvtha5nH2EHUK6WRn6qlNzqjxaRhSLyoojsKSIbEkyy74pjGB7Ix3EGGUW41X8HOERVVyWs6PadBtbBcZwGUIQ5VoFSpJsxwKxG1cFxnMbQ0MHREmXm2M8Dt4jI9wg9nq0SaY4iRAyjczUjopbjOANOEebYY4AvxE+VLxAGUJfCo5w7TvNShFv9oUDp9zV4lHPHGXQU4VY/C9g+/t4JqDhz1HGc5qWIKOdnAL8RkVZgEbBPbk7ak3T/VrFPwXKrt+YpmHMqEnNKSlhzAqQ3na/lhp0XqVxb0vMxrPkCQxanp9LkRXO35odY52nO01gWjAjyeXNvrPrWS+48GOOetfSk0/YY6WZ98mSzzM4Zled5tP/HTLYURZhjjwP2UtVhwAnArg2sg+M4DaBhikNVZ6vqA/H3m0DJHDsVuDMedhvw4UbVwXGcxlCEd+xj9H2e7A+smkjT5x37invHOk4zUYQ59gjgWBG5HxhFCB+4FP3MsePcO9ZxmokB945V1SeB3aJ8KrBXI+vgOM7yp4hFpyfE/y3AKcCPGlUHx3EagyxDgHE7Y5FtgD8DjxDWToEQ5Xxt4NNx+1rgK3lRzkVkLiH4cYlxpJdUaITMy/Qy3+llrq6q4418+qOqg+4PuG8gZV6ml/nfVmbe34BYVRzHeWfhisNxnJoZrIrjJwMs8zK9zP+2Mk0aNjjqOM47l8Ha43Acp0BccTjOICbOlxpwXHFUQT03R0SSsfFFZJWibngzknctBtu1StW31v1Rlje7e0g8Lvksl+e/PK7noFAcIjJNRLYUkfYYx6PSMan9a4nIJiKyVBADEVlXRLYXkaUCaIjINiJyMICqavZii8j7ReRzRn33Ac4szZItk70PuI4Kzn0isoWIHBz/d5TJ1o7n0ZI61wr51dVAlueDLCJJRyMRWQXC9U3I17bk1dSnwkOzqoh0lBR79oHLyWdSNl0F+RQRGSMiYyq0l/eKSEul8xCRzUnH3d0ROLFS243y9wE3icjKqv2DrojIu0VkuoisUl4fYLKItFW6BlVT7wSQgfojrPj2JHA78HPgs8DojHxq5ndrWdq9gYeBPxKWm8weu0eUXQ/8Flgl7m8BRhK8eB8Hjs6kaSH42TwI7Jqo7/axvkvJM2mfBX5QJvtArM+lwC+BtTOyDwIPEfx+zgGOBUZUyH/zWP6mmX2lAfDRleobZRsD2wCbJeRbArsnzmkPwnIXldK9DzgRGJpIdzWwViLtrsBc4IgKsp2AI4EjK8g2A7YGNqlwDfYCHiVYE64GpmXu617A8cDICnnuTogv8zPgx6W2UnaeDwDnAVcAK2ZkqwCLCWsJtVdI9yQhbk2l6/MMsFvZ/paytLcDO1SQPRXP86/A+LJz+RshoNYviM9EKW3Vz+XyesAb8Qe0A1cBW8ftDwPfBb5FWGJhb0KAoF9k0rTG/1sRYoBsFLfPBy6Ov3cAni49KIQewC5lZZ9ECDT0c0Jw5VKeL2fSjQFWB4Zn0h0PfDH+nhQfgM0JoQT+Cawbz+tWYLt43ErALcB6cftiQsiBCTGPm4HpUXZEvPH/A4wqa2j/iI3lOuCijOxDBMWzeXkDidfw7/E8rwY+VSbfM6b9DkHBfiAjGwL8GlgI7FOh4T9UatQVHu7ngJ0qyFpi434w1unksod/D8LD/0XgDuDATNq9Ypnfjml/XEpL6OE9Eu/9yjH97Hg/NgUWxPtzFBnlAewY28o2wCbAmcBBmXx3iPXZEVgnXv8V6GuHY4HfEV5CVwMdcf82wExgx7g9Mv4fBnQAPwT2jPtWiG1hXJmi2pagmG/M1Hca4aVXyvccwtTyEQR3jydiupHA14AXqEN5FK4cqlAcNwOHZRrV9rERfzbekKOAS4DLs8qD8JAfltk3ntC7GAK8O3NhVyHEQb2e8DbZLzaI4+NF35nQW/l+lL9I6AGsROjJ3BQbRCndZ+lTHP8X015GeDO8L9MQfggck1FAd8Y8RhPeNDcS3gjfBf5C5iEjBHn+AfGhied7JXBw3B4d0/wSmALcRQiadCWh8Zcewo0IvZwN4vb+wNmZcjYG7gO2jNunE3pGEzLHHBnz/TdwaNy3btw+KqMYpwHvidsHAd/KKNe9iL0WwoP4d8K6wuOBl4g9HULjv4UQQQ5CNLkD4zkNJ7SVnaNsNWAOfS+LVsJD3Zk5/88RHt6DCb3BjeM9/TR9D/KJZHpUhBfKjzPbn6GvLU0htKWzCS+qteP+YwkvmGsIbXVbggK6Glg/yn5BcPi8hvCAnwUcAkwmvCguJTzk28R7v0PmGfkTsWcW63B+5vcrhBfRg4SX2IWZ81+PECPnn8CaNT2bRSuHKpTHrsANwLaZBvAx4PLY6EYSNOovWVp5jM78nhwb5PhSY47/vwqcEn8fRngIxgNrAl+O+08g9GzOAzYgPNgvEh6aFkIvYAbh7fIeQjfxSuDwmP5dsVG8j77u5O6Eh6L0MO0H3A/cDfxP3LdTbGjfjOd7MKG3dTnwKfr3Kr5EVByZfX8mdJ23j9unxmu5CSGkwlbETzGC0lsLuJfwdhZCz2CLKB9LeChuJCjC8+L+fQgK572EHs+Z8VpcAHwynufvCT3H2wgKc4d4LVclvDnPICioKwk9nM0z53AcoeGPISiOSwiKZkPCJ99VBAV9PeFBXC+T9ruxTpcTlNdVwEkZ+VqEF8IVhLYkhF7ZH4DTCIpkAv0/cTeP9VyL8AIq9QKGxrqdGM/5XMIDPZrQOzwxHvcgwenzYMIaQxcQ2tJnCT3D8+J1OobQI/gSfS+YUwjKsKSQSm3p2JhuY0I7vzfmO5ug6NYGvgc8T1A+X4r1OoPQhr9CeA6EqFTeCYpjaGw8PyF27eP+PwAbZrZXIowBXB63NwbWib/bCArm9rj98Xhhh1Uo76aYdhLhm/bI2PhOJTw0nwKmA8eVpftdqT7A+wlv3G9k5D8ldnEz+74Rb5oQFNCKhMa+d+aYXwEfjXW+GPh+RvZH+pTjQYQu82pxeyp9CjX7IP5PvHabxu3S51Ir4a19I32fd2vH/S2Et/ChMd/OWPYOwBrAjHj8GYTATOcRxhnOBv4FHB3TrUpQIocRFNhXgeNj2mmE7/HPlj0UmxF6a6vH7S8Q3sr3EnqepW72vXH/i8ABhN7hbwjjALMJ7ecDBGXzFfrGv+6O9/c9mWt0AjCf8Hlxb6xrW5RtSngxPEzo9j9Gn/Kflsn3L4QXzLoEBfO5eN6LgTeibDtCj+WoTLq/Ej6N9iR8zj5AUJQl+WxCG80qyOOALsKn2DmEF86q8fe+Md2dsa6fJPRAL4rXp53wcj6rpueyaMVQpfJYMTbcm+NFPjRehJXLjhtHeNifjI1hcpn8EuD/Ed7s76FMuxLGUO6nb6D0GwQt/f64vSOwaoX6ldKtnFFUh8TG8Yn4dx9l3cGY7i4yg7qEb/ifEbrOH4gNZ0r2YYq/vw/0ANdk9n2T8EY5nNBDmkF4O25G37f13oSH+znCg/5wpt4tsZ5vxQZ8A3Ggj/CJ9/aYUmx4W8V7c27Mqzfm9yrwkVjuvpl0M+I92IKgTJ4B/jce+xbhbVzqpbVl6tsNzMyc5/B4fb8a011FUCL7ER7QUwi9oicIn2O/jtunE14ILxHe3B8gPMxz6HvhlMbGdic8pG8C12XKPgB4ndCzuD9ei0vL0m5EUPQz4/kOJyi0Xvp6D/+i7zNq+7J0swifIysQlOZlBIV3CqFtX5JJWyrz6/G4S4g9UULPZU4m31eBczNttKScP0loc0N4p/Q4Mjesg/DgXhkvzkaJ475A5hMg7pOY/l8ERbB2WZohhIf7Mfpr8lXJjHiz9MCiED5THgfWrVCXjQkDdWdl61N2zNVExRC3VyB0W/9E+J7foEKaowlvxK/HazEjIzuD8Ea7jKB05gFrRNmK9I0LvUR46Epvy1bC58BLhE+cV+g/6DY6k/YPUb56lJ0VH4oLYn1+T7SWEHqCpXR/jI23lO7IeOwThJ7OPOCGTJml+h5PUHS3ZWRHEgZlv034VPkP/T8ptiIohYMIynka8Nso24/wBj6f8PBvR9/419Yx3VaxPttE2dCYdtd4fe8n9CayY2dbxXodSxik3aZ0DQm9mG9n6jeeoJg7MnX9RCbdb+JxnfG63kDoaZSXWarvdoSHf0rpGhIU8muET7FH47E3ZM6ljfDZ/zyZdv+OUhyZC95KYvQ3NrTbgPUT8sOo/IC3E7qG0xLpKmph+kbV16njPEzNTojHWtGEShhM24r+4ztZ5XEE4fv5MkJ3Nzv2M4kwPvBwvFaXl+X9O8KDvDlLjxutRlDMD8Z8fxH3t8TGnq3PFWVlHkUYY7q1rK7bEHqQZybKnBQflHMJyiWb73cIiveWbH2irIMwZnAvfd/+fwcmxja0bnxwVszIxkfZVMJLam2WHhubRlBE61A2dha3NyQopHUzsnFR1hnbWqV06xN6kdMzslLPdzJBcY9IpF09HrdSRjYpyjaL57FKhXNZjaBYaxoY1cGoOKp4IJeaM1DtwzpY/+gb3ymNNaxb1piyYz8bEt5O4yrI1ia8wacn0q5D+BRYK5HvuxPp3k14C7+rQl3XBzornEsp7Sax0bdkZFdG2btiuR0V0q1H+BRZhaXHuA4ijIEMqyD7OEFJjaggO4TwObhC3C6XH0zo8Y3OyP6QyfeCRL4HERTn6ITsx8TxuER9f5I4l0MI402VZAcRxqCWmrdSVZsrutH733JTHqXxnacI5rXJFWSlsZ9JFWRPx7/UuNFTUb6KkW+lMkvpJlZT1wr5ps7l6Zwy+51nlF9CZozLkK1frazGtPWWudRnbg35Vl1mTe2t6Abvf8vvjwrjO8sqa1S+A1kmxhhXvbJG5VtEmXW1taIbu/8tnz+M8Z16ZY3Kt4gyo/wwKoxxLYusUfkWUWYtfx7I5x2EiAxV1UXLU9aofAsqUzTR4OuVNSrfIsqsBVccjuPUzKBwq3ccp7lwxeE4Ts244nAcp2ZccTiOUzOuON4BiEiPiDwoIo+KyDUiMnwZ8rpERPaLvy8UkenGsTuISMWwdzllPCsi46rdX3bM/BrLOk1EvlhrHR0bVxzvDBaq6oaquh7B8/XorLCKgLcVUdVPqurjxiE7kIiX6byzccXxzuPPwFqxN/BnEbkBeFxEWkXkuyLyNxF5WEQ+BcGuLyL/KyJPicjvCYFriLI7RGST+Ht3EXlARB4SkdtFZApBQX0h9na2FZHxIvKrWMbfRGTrmHYlEblVRB4TkQsJsxhNROR6Ebk/pjmqTHZ23H+7iIyP+9YUkd/FNH8WkXWWx8V0EizrDDL/K/4PmB//txFiTxxD6A0soM+l/ij6Ip0NIcTdWIPgQXobwZNyEsGVfL943B0EJ7PxhDgfpbzGxv+nEcMkxu1fANvE36sBT8Tf5wKnxt97AUqMnFV2Hs/SF1GrVMYwgkt4KWKbAh+Pv08F/jf+vp2+yFib0+dc1q+O/rd8/urqwjpNxzAReTD+/jN9QXbuVdV/x/27AeuXxi8IsTfWJnjKzlDVHmCWiPyhQv5bAHeW8lLV/yTqsQswPROJf7SIjIxlfCim/a2IvFbFOX1WRPaNv1eNdX2VEPfjqrj/cuDaWMZWwDWZsisuKeAsH1xxvDNYqKobZnfEB2hBdhfwGVW9pey4PZdjPVoIMUr7TfmWGpd3EZEdCEpoS1V9S0TuIISQrITGcueVXwOncfgYx38PtwDHiEg7gIhMjQvy3Al8JI6BTCQEsCnnbmA7EVkjph0b979JCDhU4lZCDE3icaUH+U5CpClEZA+CM5rFGOC1qDTWIfR4SrQQIngR87xLVd8A/i0i+8cyREQ2yCnDWQZccfz3cCEhxOEDIvIoIThMG2ENln9E2c8JsUb7oapzCWMk14rIQ/R9KtwI7FsaHCWEPNwkDr4+Tp915+sExfMY4ZPl+Zy6/g5oE5EnCIFx7s7IFgCbxXPYiRAXFkJAm0/E+j1GiL7uNAh3cnMcp2a8x+E4Ts244nAcp2ZccTiOUzOuOBzHqRlXHI7j1IwrDsdxasYVh+M4NfP/AVzdBkAeYladAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "tags": [],
            "needs_background": "light"
          }
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 3600x3600 with 0 Axes>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RsdkxZPD-Op4",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "outputId": "2ac83b6c-b5e9-429f-866b-4d6b8922be69"
      },
      "source": [
        "# confusion matrix\n",
        "pd.crosstab(test_y, preds)"
      ],
      "execution_count": 60,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th>col_0</th>\n",
              "      <th>0</th>\n",
              "      <th>1</th>\n",
              "      <th>2</th>\n",
              "      <th>3</th>\n",
              "      <th>4</th>\n",
              "      <th>5</th>\n",
              "      <th>6</th>\n",
              "      <th>7</th>\n",
              "      <th>8</th>\n",
              "      <th>9</th>\n",
              "      <th>10</th>\n",
              "      <th>11</th>\n",
              "      <th>12</th>\n",
              "      <th>13</th>\n",
              "      <th>14</th>\n",
              "      <th>15</th>\n",
              "      <th>16</th>\n",
              "      <th>17</th>\n",
              "      <th>18</th>\n",
              "      <th>19</th>\n",
              "      <th>20</th>\n",
              "      <th>21</th>\n",
              "      <th>22</th>\n",
              "      <th>23</th>\n",
              "      <th>24</th>\n",
              "      <th>25</th>\n",
              "      <th>26</th>\n",
              "      <th>27</th>\n",
              "      <th>28</th>\n",
              "      <th>29</th>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>row_0</th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "      <th></th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>48</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>7</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>32</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>12</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>11</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>3</td>\n",
              "      <td>66</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>15</td>\n",
              "      <td>12</td>\n",
              "      <td>6</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>90</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>3</td>\n",
              "      <td>15</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>14</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>167</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>43</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>16</td>\n",
              "      <td>9</td>\n",
              "      <td>3</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>10</td>\n",
              "      <td>88</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>11</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>14</td>\n",
              "      <td>3</td>\n",
              "      <td>15</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>6</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>19</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>113</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>11</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>166</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>8</th>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>13</td>\n",
              "      <td>91</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>10</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>9</th>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>83</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>8</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>42</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10</th>\n",
              "      <td>3</td>\n",
              "      <td>6</td>\n",
              "      <td>11</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>18</td>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>0</td>\n",
              "      <td>57</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>29</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>14</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>11</th>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>15</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>64</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>25</td>\n",
              "      <td>0</td>\n",
              "      <td>15</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>16</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>12</th>\n",
              "      <td>5</td>\n",
              "      <td>18</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>97</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>13</th>\n",
              "      <td>9</td>\n",
              "      <td>16</td>\n",
              "      <td>7</td>\n",
              "      <td>3</td>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>10</td>\n",
              "      <td>14</td>\n",
              "      <td>13</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>46</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>14</th>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>111</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>15</th>\n",
              "      <td>4</td>\n",
              "      <td>9</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>56</td>\n",
              "      <td>0</td>\n",
              "      <td>20</td>\n",
              "      <td>5</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "      <td>17</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>7</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>16</th>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>20</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>89</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>17</th>\n",
              "      <td>2</td>\n",
              "      <td>12</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>20</td>\n",
              "      <td>0</td>\n",
              "      <td>86</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>14</td>\n",
              "      <td>1</td>\n",
              "      <td>10</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>18</th>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>5</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>90</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>17</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>19</th>\n",
              "      <td>4</td>\n",
              "      <td>10</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "      <td>5</td>\n",
              "      <td>9</td>\n",
              "      <td>7</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>11</td>\n",
              "      <td>46</td>\n",
              "      <td>7</td>\n",
              "      <td>22</td>\n",
              "      <td>4</td>\n",
              "      <td>11</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>20</th>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>5</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>10</td>\n",
              "      <td>71</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>10</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>12</td>\n",
              "      <td>12</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21</th>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>19</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>108</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>22</th>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>14</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>79</td>\n",
              "      <td>0</td>\n",
              "      <td>16</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23</th>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>1</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>112</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>24</th>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>8</td>\n",
              "      <td>10</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>27</td>\n",
              "      <td>0</td>\n",
              "      <td>84</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>25</th>\n",
              "      <td>3</td>\n",
              "      <td>6</td>\n",
              "      <td>14</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>28</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>14</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>57</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>2</td>\n",
              "      <td>12</td>\n",
              "      <td>4</td>\n",
              "      <td>1</td>\n",
              "      <td>9</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>6</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>93</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>16</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>27</th>\n",
              "      <td>2</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>9</td>\n",
              "      <td>6</td>\n",
              "      <td>1</td>\n",
              "      <td>5</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>6</td>\n",
              "      <td>6</td>\n",
              "      <td>8</td>\n",
              "      <td>11</td>\n",
              "      <td>4</td>\n",
              "      <td>7</td>\n",
              "      <td>8</td>\n",
              "      <td>4</td>\n",
              "      <td>4</td>\n",
              "      <td>6</td>\n",
              "      <td>23</td>\n",
              "      <td>5</td>\n",
              "      <td>9</td>\n",
              "      <td>10</td>\n",
              "      <td>12</td>\n",
              "      <td>18</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>28</th>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>8</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>130</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>29</th>\n",
              "      <td>6</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>9</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>12</td>\n",
              "      <td>4</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>7</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>2</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>115</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "col_0  0   1   2    3   4   5    6    7   ...  22   23  24  25  26  27   28   29\n",
              "row_0                                     ...                                   \n",
              "0      48   5   3    2   5   7    6    4  ...   2    8   2   1   5   1    0   11\n",
              "1       3  66   3    0   3   0    0    5  ...   4    1   1   3  11   0    0    9\n",
              "2       2   5  90    0   0   0   17    1  ...   0   14   1   3   3   0    4    2\n",
              "3       0   0   0  167   0   0    0    4  ...   0    0   1   1   0   1    1    2\n",
              "4       0  13   1    4  43  10    1    8  ...   7    4   5   1  16   9    3    7\n",
              "5       6   3   0    0  10  88    3    0  ...  14    3  15   1   3   1    0    2\n",
              "6       0   0  19    4   2   0  113    2  ...   1   11   3   1   0   0    9    0\n",
              "7       0   1   0    0   1   0    0  166  ...   3    2   0   0   4   0    0    1\n",
              "8      10   0   3    3   4   1    2   13  ...   2   10   1   1   8   0    0    6\n",
              "9       1   6   1    2   0   0    0    1  ...   3    3   0   4   3   0    0    2\n",
              "10      3   6  11    2   2   1   18    0  ...   2   29   0   0  14   0    5    4\n",
              "11      1   2   1    0   5   1    0   15  ...   1   11   0  16  10   0    0    2\n",
              "12      5  18   1    0   2   0    0    2  ...   0    4   2   0   5   1    1   12\n",
              "13      9  16   7    3   9   6   10   14  ...   3    1   6   0   7   3    0    2\n",
              "14      1   4  12    0   0   0    5    0  ...   2    4   0   1   0   0    5    1\n",
              "15      4   9   2    2   8   2    1    2  ...  17    1   6   2   5   2    2    7\n",
              "16      0   3   6    0   1   0    8    3  ...   1   24   0   3   0   0    6    0\n",
              "17      2  12   2    0   7   2    0    1  ...  14    1  10   3   0   2    0    3\n",
              "18      0   6   5    1   5   0    0    6  ...   3    0   1  17   1   0    1    1\n",
              "19      4  10  12    0   4   0    3    2  ...   4   11   0   6   3   3    4    0\n",
              "20      4   1   8    2   2   1    2    4  ...   1   10   2   3   7   0   12   12\n",
              "21      2   2   2    1   5   2    0    2  ...   5    4   2   3   0   0    1    0\n",
              "22      0   9   0    0   5   9    1    2  ...  79    0  16   3   1   3    0    2\n",
              "23      1   6   4    1   5   0    4    2  ...   2  112   1   1  10   0    0    8\n",
              "24      0   6   3    0   8  10    5    1  ...  27    0  84   0   5   1    1    2\n",
              "25      3   6  14    2   0   0    7    5  ...   3    4   2  57   4   0    0    0\n",
              "26      2  12   4    1   9   1    1    0  ...   1    6   3   4  93   2    1   16\n",
              "27      2   8   0    0   9   6    1    5  ...  23    5   9  10  12  18    4    3\n",
              "28      0   0   4    3   2   1    8    1  ...   1    2   0   1   3   0  130    2\n",
              "29      6   4   0    3   4   0    0    2  ...   2    2   3   0   6   0    0  115\n",
              "\n",
              "[30 rows x 30 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 60
        }
      ]
    }
  ]
}