MSHA_with_HF_transformers_and_Pretraining.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "MSHA with HF transformers and Pretraining",
      "provenance": [],
      "collapsed_sections": [],
      "toc_visible": true,
      "authorship_tag": "ABX9TyP4aUWz1HUWObVDp9Hu2q+w",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "8b216466e4614ca5980da1c0d4e6b9dc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_0a008986de3e4af4a3422c9b434e6186",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_22cfea7268f6495f83b0b91eb8e08259",
              "IPY_MODEL_81ef245852ac4eca8f4ddf80cc2351fb"
            ]
          }
        },
        "0a008986de3e4af4a3422c9b434e6186": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "22cfea7268f6495f83b0b91eb8e08259": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_933876daacd84fc7b4a9188db2063a81",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 442,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1d223995b8f346e3bea7c6f42d40e742"
          }
        },
        "81ef245852ac4eca8f4ddf80cc2351fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_4928c5752f36499383d02d1af8ae1c07",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442/442 [00:00&lt;00:00, 2.22kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_76190a8472204ac08fc41379a5c3f34b"
          }
        },
        "933876daacd84fc7b4a9188db2063a81": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1d223995b8f346e3bea7c6f42d40e742": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4928c5752f36499383d02d1af8ae1c07": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "76190a8472204ac08fc41379a5c3f34b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9e473fd8a59f47ee91cadec9b1a068d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_d887d4e93a1f4cbd958087a43516fe9f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_b72a48621a7e48f98716f3a21e397365",
              "IPY_MODEL_3782809b46cd4d108f9157b69bba5713"
            ]
          }
        },
        "d887d4e93a1f4cbd958087a43516fe9f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b72a48621a7e48f98716f3a21e397365": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_00b2c227dbf3425bb8b6c58eec2f5af3",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 267967963,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 267967963,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3945fafd4667445d93f7509df0e0a225"
          }
        },
        "3782809b46cd4d108f9157b69bba5713": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_48f9c9d1df5042f3a9f1a5f6b71044e7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 268M/268M [00:03&lt;00:00, 68.6MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1dd26608e44c4d28a9d9a709187ae722"
          }
        },
        "00b2c227dbf3425bb8b6c58eec2f5af3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3945fafd4667445d93f7509df0e0a225": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "48f9c9d1df5042f3a9f1a5f6b71044e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1dd26608e44c4d28a9d9a709187ae722": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9675712624d7402a8cb45acbeccfb216": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_854fd456ba8b4150845c8f78fc4bca38",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_4936102d50014b8fa3307544fe2b0871",
              "IPY_MODEL_282b631e5edb4d9896f79739bfb5b6c2"
            ]
          }
        },
        "854fd456ba8b4150845c8f78fc4bca38": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4936102d50014b8fa3307544fe2b0871": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1a50e42be6714e93a7e3e9a4af011707",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_cdcd767796d84349803da828aa2bdf6d"
          }
        },
        "282b631e5edb4d9896f79739bfb5b6c2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_dce27a5ef6d24282868c9c1ac3aa461b",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:00&lt;00:00, 3.41MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_c16a7fedfef64abb91d009e46ef2d770"
          }
        },
        "1a50e42be6714e93a7e3e9a4af011707": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "cdcd767796d84349803da828aa2bdf6d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "dce27a5ef6d24282868c9c1ac3aa461b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "c16a7fedfef64abb91d009e46ef2d770": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ameasure/colab_tutorials/blob/master/MSHA_with_HF_transformers_and_Pretraining.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bIpB9LhIPGpH"
      },
      "source": [
        "# Advanced Text Classification with Transformers and Pretraining\n",
        "\n",
        "This tutorial describes how to pretrain an already pretrained language model on domain specific data, and then finetune it for classification."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jEHLnpfG4b3G"
      },
      "source": [
        "## Install Libraries"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGYWeOkaO-_n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8ccf5d98-b373-4e70-d144-47a97e16874e"
      },
      "source": [
        "!pip install transformers==3.4.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers==3.4.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/2c/4e/4f1ede0fd7a36278844a277f8d53c21f88f37f3754abf76a5d6224f76d4a/transformers-3.4.0-py3-none-any.whl (1.3MB)\n",
            "\u001b[K     |████████████████████████████████| 1.3MB 18.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (3.0.12)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (20.4)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 53.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (2.23.0)\n",
            "Collecting tokenizers==0.9.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7c/a5/78be1a55b2ac8d6a956f0a211d372726e2b1dd2666bb537fea9b03abd62c/tokenizers-0.9.2-cp36-cp36m-manylinux1_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 36.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: protobuf in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (3.12.4)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (2019.12.20)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (1.18.5)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers==3.4.0) (0.7)\n",
            "Collecting sentencepiece!=0.1.92\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/e5/2d/6d4ca4bef9a67070fa1cac508606328329152b1df10bdf31fb6e4e727894/sentencepiece-0.1.94-cp36-cp36m-manylinux2014_x86_64.whl (1.1MB)\n",
            "\u001b[K     |████████████████████████████████| 1.1MB 54.4MB/s \n",
            "\u001b[?25hRequirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==3.4.0) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from packaging->transformers==3.4.0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.4.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==3.4.0) (0.17.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.4.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.4.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.4.0) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==3.4.0) (1.24.3)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.6/dist-packages (from protobuf->transformers==3.4.0) (50.3.2)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893257 sha256=6588cb3d8c94eaaf88f095b7ee53fd39c49760e650f69cb29cfdbf97150efce5\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: sacremoses, tokenizers, sentencepiece, transformers\n",
            "Successfully installed sacremoses-0.0.43 sentencepiece-0.1.94 tokenizers-0.9.2 transformers-3.4.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cHPZyxbZQOv4"
      },
      "source": [
        "## Download and Prepare Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHwxv-7DPFUa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "1c376d80-7d17-4441-feae-7a7ae4420e0e"
      },
      "source": [
        "!wget 'https://github.com/ameasure/autocoding-class/raw/master/msha.xlsx'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2020-11-20 20:59:58--  https://github.com/ameasure/autocoding-class/raw/master/msha.xlsx\n",
            "Resolving github.com (github.com)... 140.82.114.3\n",
            "Connecting to github.com (github.com)|140.82.114.3|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/ameasure/autocoding-class/master/msha.xlsx [following]\n",
            "--2020-11-20 20:59:58--  https://raw.githubusercontent.com/ameasure/autocoding-class/master/msha.xlsx\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.0.133, 151.101.64.133, 151.101.128.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.0.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4183086 (4.0M) [application/octet-stream]\n",
            "Saving to: ‘msha.xlsx.2’\n",
            "\n",
            "msha.xlsx.2         100%[===================>]   3.99M  --.-KB/s    in 0.06s   \n",
            "\n",
            "2020-11-20 20:59:58 (71.3 MB/s) - ‘msha.xlsx.2’ saved [4183086/4183086]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ujGe33a0QVPV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 246
        },
        "outputId": "ac265821-5c38-4b9f-bb48-8cab38bff488"
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn import preprocessing\n",
        "\n",
        "# read in the data\n",
        "df = pd.read_excel('msha.xlsx')\n",
        "df['ACCIDENT_YEAR'] = df['ACCIDENT_DT'].dt.year\n",
        "# convert part codes to numeric indicators\n",
        "labeler = preprocessing.LabelEncoder()\n",
        "labeler.fit(df['INJ_BODY_PART'])\n",
        "df['PART_CODE'] = labeler.transform(df['INJ_BODY_PART'])\n",
        "# separate pretraining, training, and validation data\n",
        "# To simulate the common scenario where we have more unlabeled pretraining data\n",
        "# than labeled data use all available 2010 and 2011 data for pretraining and\n",
        "# only a small sample of that for supervised training.\n",
        "df_pretrain = df[df['ACCIDENT_YEAR'].isin([2010, 2011])].copy().reset_index(drop=True)\n",
        "df_train = df_pretrain.sample(1000).copy().reset_index(drop=True)\n",
        "df_valid = df[df['ACCIDENT_YEAR'] == 2012].copy().sample(1000).reset_index(drop=True)\n",
        "# show the rseults\n",
        "print('n_classes:', len(df['INJ_BODY_PART'].unique()))\n",
        "print('training rows:', len(df_train))\n",
        "print('validation rows:', len(df_valid))\n",
        "df[['INJ_BODY_PART', 'PART_CODE', 'NARRATIVE', 'ACCIDENT_YEAR']].head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "n_classes: 46\n",
            "training rows: 1000\n",
            "validation rows: 1000\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>INJ_BODY_PART</th>\n",
              "      <th>PART_CODE</th>\n",
              "      <th>NARRATIVE</th>\n",
              "      <th>ACCIDENT_YEAR</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)</td>\n",
              "      <td>35</td>\n",
              "      <td>Cleaning out Gabion Grizzly,  Rocks get Jammed...</td>\n",
              "      <td>2010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)</td>\n",
              "      <td>35</td>\n",
              "      <td>Injured was walking in the pit area, stepped o...</td>\n",
              "      <td>2010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>HIPS (PELVIS/ORGANS/KIDNEYS/BUTTOCKS)</td>\n",
              "      <td>22</td>\n",
              "      <td>Employee, parked s/c on grade at 16-Block #3 E...</td>\n",
              "      <td>2012</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ANKLE</td>\n",
              "      <td>1</td>\n",
              "      <td>Contractor employee working as a carpenter mis...</td>\n",
              "      <td>2013</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>16</td>\n",
              "      <td>The employee's finger was pinched between the ...</td>\n",
              "      <td>2011</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                             INJ_BODY_PART  ...  ACCIDENT_YEAR\n",
              "0  SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)  ...           2010\n",
              "1  SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)  ...           2010\n",
              "2    HIPS (PELVIS/ORGANS/KIDNEYS/BUTTOCKS)  ...           2012\n",
              "3                                    ANKLE  ...           2013\n",
              "4                          FINGER(S)/THUMB  ...           2011\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C00ibcTYDWdy"
      },
      "source": [
        "# Domain-Specific Language Model Pretraining (Optional)\n",
        "\n",
        "Recent research such as [ULMFiT](https://arxiv.org/abs/1801.06146) and [Don't Stop Pretraining](https://arxiv.org/abs/2004.10964) suggests models pretrained on general-purpose language modelling can often be improved by additional task-specific language modeling. We illustrate how to do this this below by further pretraining an already pretrained model on MSHA data. Note that because we performing the language model pretraining task, we do not even use the part codes assigned to the individual cases. This approach allows us to make use of large amounts of unlabeled task-specific data."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vTYqVEF2Q4z9"
      },
      "source": [
        "### Specify the Language Model\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0bT9nFG-QY0n",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 163,
          "referenced_widgets": [
            "8b216466e4614ca5980da1c0d4e6b9dc",
            "0a008986de3e4af4a3422c9b434e6186",
            "22cfea7268f6495f83b0b91eb8e08259",
            "81ef245852ac4eca8f4ddf80cc2351fb",
            "933876daacd84fc7b4a9188db2063a81",
            "1d223995b8f346e3bea7c6f42d40e742",
            "4928c5752f36499383d02d1af8ae1c07",
            "76190a8472204ac08fc41379a5c3f34b",
            "9e473fd8a59f47ee91cadec9b1a068d8",
            "d887d4e93a1f4cbd958087a43516fe9f",
            "b72a48621a7e48f98716f3a21e397365",
            "3782809b46cd4d108f9157b69bba5713",
            "00b2c227dbf3425bb8b6c58eec2f5af3",
            "3945fafd4667445d93f7509df0e0a225",
            "48f9c9d1df5042f3a9f1a5f6b71044e7",
            "1dd26608e44c4d28a9d9a709187ae722",
            "9675712624d7402a8cb45acbeccfb216",
            "854fd456ba8b4150845c8f78fc4bca38",
            "4936102d50014b8fa3307544fe2b0871",
            "282b631e5edb4d9896f79739bfb5b6c2",
            "1a50e42be6714e93a7e3e9a4af011707",
            "cdcd767796d84349803da828aa2bdf6d",
            "dce27a5ef6d24282868c9c1ac3aa461b",
            "c16a7fedfef64abb91d009e46ef2d770"
          ]
        },
        "outputId": "26fc1d18-8515-4197-a420-245f147190c1"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import AutoModelForMaskedLM, AutoConfig, AutoTokenizer\n",
        "\n",
        "#model_name = 'roberta-base'\n",
        "model_name = 'distilbert-base-uncased'\n",
        "config = AutoConfig.from_pretrained(model_name)\n",
        "model = AutoModelForMaskedLM.from_pretrained(model_name)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8b216466e4614ca5980da1c0d4e6b9dc",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9e473fd8a59f47ee91cadec9b1a068d8",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=267967963.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "9675712624d7402a8cb45acbeccfb216",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5AH5uRjFfOTl"
      },
      "source": [
        "Where do we get those input_ids and attention_mask? They're generated by the tokenizer."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PJ7nnjuqMFs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4c541aa6-e812-4ade-9d34-9ea8954606fa"
      },
      "source": [
        "tokenizer.encode_plus('a man fell while lifting a ladder')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [101, 1037, 2158, 3062, 2096, 8783, 1037, 10535, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SDaR_X6fW_9-"
      },
      "source": [
        "### Test the language model\n",
        "\n",
        "A language model attempts to predict missing words (tokens) using the surrounding words. We can verify that our model is indeed pretrained by testing it out. One simple way is by using the fill_mask pipeline, as follows. Note, in this case we are asking the model to complete the prompt `\"the worker sprained his [blank]\"`. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MCl1scbJW7ti",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "924f79d2-f7bd-473e-868b-3e443d1b5898"
      },
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "fill_mask = pipeline(\n",
        "    \"fill-mask\",\n",
        "    model=model,\n",
        "    tokenizer=tokenizer\n",
        ")\n",
        "\n",
        "fill_mask(f'The worker sprained his {tokenizer.mask_token}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'score': 0.33903393149375916,\n",
              "  'sequence': '[CLS] the worker sprained his ankle [SEP]',\n",
              "  'token': 10792,\n",
              "  'token_str': 'ankle'},\n",
              " {'score': 0.07981275767087936,\n",
              "  'sequence': '[CLS] the worker sprained his wrist [SEP]',\n",
              "  'token': 7223,\n",
              "  'token_str': 'wrist'},\n",
              " {'score': 0.04735306650400162,\n",
              "  'sequence': '[CLS] the worker sprained his knee [SEP]',\n",
              "  'token': 6181,\n",
              "  'token_str': 'knee'},\n",
              " {'score': 0.04560253396630287,\n",
              "  'sequence': '[CLS] the worker sprained his foot [SEP]',\n",
              "  'token': 3329,\n",
              "  'token_str': 'foot'},\n",
              " {'score': 0.044954314827919006,\n",
              "  'sequence': '[CLS] the worker sprained his neck [SEP]',\n",
              "  'token': 3300,\n",
              "  'token_str': 'neck'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "In-0sOxsxxlR"
      },
      "source": [
        "### Convert Data into PyTorch Datasets / Samplers / Loaders\n",
        "\n",
        "PyTorch provides four optional utilities to assist with managing data, especially when that data is large or requires expensive processing that we would like to parellelize and conduct on-the-fly. These are as follows:\n",
        "* PyTorch Dataset - a representation of the input data accessible either by index or by iteration\n",
        "* PyTorch DataSampler - a mechanism for sampling indexes from the Dataset (typically either sequential or random)\n",
        "* PyTorch collate_fn - a functon for collating samples into batches\n",
        "* PyTorch DataLoader - a mechanism that combines the other mechanisms to load batches onto the GPU. In particular this allows the preparation of batches while the GPU is processing so they are ready the instant the GPU becomes available for the next one.\n",
        "\n",
        "An added advantage of using these data structures is that other libraries (like Transformers, and Pytorch-Lightning) are designed to work with them."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VVSSx5iBxv2C"
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import Dataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "class CustomDataset(Dataset):\n",
        "  def __init__(self, dataframe, tokenizer, max_len, input_field, target_field=None):\n",
        "    self.tokenizer = tokenizer\n",
        "    self.dataframe = dataframe\n",
        "    self.target_field = target_field\n",
        "    self.input_field = input_field\n",
        "    self.max_len = max_len\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.dataframe)\n",
        "\n",
        "  def __getitem__(self, index):\n",
        "    input = self.dataframe[self.input_field][index]\n",
        "    inputs = self.tokenizer.encode_plus(\n",
        "        input, None, add_special_tokens=True, max_length=self.max_len, \n",
        "        padding='max_length', truncation=True, return_token_type_ids=False)\n",
        "    # if we know the code, i.e. we're using this for training, add the target\n",
        "    if self.target_field:\n",
        "        inputs['labels'] = torch.tensor(self.dataframe[self.target_field][index], dtype=torch.long)\n",
        "    return inputs   "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qI4SO9inbw7B"
      },
      "source": [
        "train_dataset = CustomDataset(dataframe=df_pretrain, tokenizer=tokenizer, max_len=200,\n",
        "                              input_field='NARRATIVE')\n",
        "valid_dataset = CustomDataset(dataframe=df_valid, tokenizer=tokenizer, max_len=200,\n",
        "                              input_field='NARRATIVE')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "poG1K6xx-HoC"
      },
      "source": [
        "Example of retrieving a row of data from our dataset by index"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jh6IYjWHGOil",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "576bf7d6-e744-4237-eeb5-c3392477004f"
      },
      "source": [
        "train_dataset[0]"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': [101, 9344, 2041, 11721, 26282, 2078, 24665, 29266, 1010, 5749, 2131, 21601, 1998, 7861, 22086, 4402, 2038, 2000, 2593, 29198, 2030, 5245, 1996, 5749, 2041, 1997, 1996, 24665, 29266, 1012, 7904, 2001, 2478, 1037, 22889, 24225, 8691, 2000, 2131, 1996, 5749, 4558, 1998, 2766, 2242, 1999, 2010, 2157, 3244, 1012, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 18
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K8brn_rR-Obm"
      },
      "source": [
        "Example of sampling an index from the dataset using the sampler"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5EtNllQQ8qoW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e075d5ec-deba-4e2d-cc8d-9afa2c07792a"
      },
      "source": [
        "sampler = RandomSampler(train_dataset)\n",
        "sampled_index = sampler.__iter__().__next__()\n",
        "print(sampled_index)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "7409\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "C3YsL4d9_Pbp"
      },
      "source": [
        "### Collator\n",
        "\n",
        "The job of the collator is to group the sampled rows into batches. In the case of language model pretraining we give it the added role of assembling \"targets\"\n",
        "for prediction, in this case the words that we want the model to predict from the context. By default this collator produces BERT style masked-language modeling targets, i.e. 15% of the tokens are chosen as prediction targets (i.e. not -100), of these 80% are replaced with a [mask] token, 10% with a random word, and 10% with the original word. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HjGkqyWK91lw",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d045dbb2-0c17-48be-c360-00f3f6cc378d"
      },
      "source": [
        "from transformers import DataCollatorForLanguageModeling\n",
        "\n",
        "collator = DataCollatorForLanguageModeling(tokenizer=tokenizer)\n",
        "collator([train_dataset[0]])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([[  101,  9344,  2041, 11721, 26282,  2078, 24665, 29266, 14823,   103,\n",
              "           2131, 21601,  1998,  7861,   103,  4402,  2038,  2000,  2593, 29198,\n",
              "           2030,  5245,  1996,  5749,  2041,  1997,   103, 24665,   103,  1012,\n",
              "           7904,  2001,  2478,   103, 22889, 24225,  8691,  2000,  2131,  1996,\n",
              "           5749,  4558,  1998,  2766,   103,  1999,  2010,  2157,  3244,   103,\n",
              "            102,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0,\n",
              "              0,     0,     0,     0,     0,     0,     0,     0,     0,     0]]),\n",
              " 'labels': tensor([[ -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  1010,  5749,\n",
              "           -100,  -100,  -100,  -100, 22086,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  1996,  -100, 29266,  -100,\n",
              "           -100,  -100,  -100,  1037,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  4558,  -100,  -100,  2242,  -100,  -100,  -100,  -100,  1012,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,\n",
              "           -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100,  -100]])}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qelYKsus-YKw"
      },
      "source": [
        "batch_size = 32\n",
        "train_loader = DataLoader(train_dataset, batch_size=batch_size, \n",
        "                          sampler=RandomSampler(train_dataset), collate_fn=collator)\n",
        "valid_loader = DataLoader(valid_dataset, batch_size=batch_size, \n",
        "                          sampler=SequentialSampler(valid_dataset), collate_fn=collator)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSIetN7yLZ4L"
      },
      "source": [
        "Example of pulling a batch from our dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wfn5cOCVLYjV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7c5aa88a-87e7-428f-b5c5-d70b43bfd5ae"
      },
      "source": [
        "batch = train_loader.__iter__().__next__()\n",
        "for k, v in batch.items():\n",
        "  batch[k] = v.to(torch.device('cuda'))\n",
        "print(batch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': tensor([[  101,  3384, 27546,  ...,     0,     0,     0],\n",
            "        [  101,   103,  2001,  ...,     0,     0,     0],\n",
            "        [  101, 25212,  2988,  ...,     0,     0,     0],\n",
            "        ...,\n",
            "        [  101,  2096,  4895,  ...,     0,     0,     0],\n",
            "        [  101,  7904,   103,  ...,     0,     0,     0],\n",
            "        [  101,  2096,  5094,  ...,     0,     0,     0]], device='cuda:0'), 'labels': tensor([[-100, -100, -100,  ..., -100, -100, -100],\n",
            "        [-100, 6778, -100,  ..., -100, -100, -100],\n",
            "        [-100, -100, -100,  ..., -100, -100, -100],\n",
            "        ...,\n",
            "        [-100, -100, -100,  ..., -100, -100, -100],\n",
            "        [-100, -100, 2001,  ..., -100, -100, -100],\n",
            "        [-100, -100, -100,  ..., -100, -100, -100]], device='cuda:0')}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x1WazxT245Y-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9606b5cb-5b02-4aad-de0b-ef00d535166d"
      },
      "source": [
        "batch['labels'].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([32, 200])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 23
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kr7yITubThlD",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0d1851bb-c226-413c-a6a6-56a102cc7c98"
      },
      "source": [
        "import transformers\n",
        "from sklearn.metrics import accuracy_score, f1_score\n",
        "\n",
        "model = model.to(torch.device('cuda'))\n",
        "optimizer = transformers.AdamW(params=model.parameters(), lr=1e-4)\n",
        "\n",
        "for epoch in range(2):\n",
        "  print(f'Epoch: {epoch}')\n",
        "  training_loss = []\n",
        "  # set the model to training mode so things like dropout behave correctly\n",
        "  model.train()\n",
        "  for idx, batch in enumerate(train_loader):\n",
        "    # send the batch to cuda\n",
        "    for k, v in batch.items():\n",
        "      batch[k] = v.to(torch.device('cuda'))\n",
        "    # calculate the model predictions on our training batch\n",
        "    loss, pred = output = model(**batch)\n",
        "    # calculate model loss, i.e. how well the predictions match the labels\n",
        "    # model already calculates the loss but...\n",
        "    # loss = criteria(output[1], batch['labels'])\n",
        "    # calculate change in loss with respect to parameters (i.e. gradient)\n",
        "    loss.backward()\n",
        "    training_loss.append(loss)\n",
        "    # adjust the parameters in the direction that reduces loss as measured by gradient\n",
        "    optimizer.step()\n",
        "    # zero out the gradient as we're now moving on to the next training batch\n",
        "    optimizer.zero_grad()\n",
        "  print(f'training_loss {torch.tensor(training_loss).mean()}')\n",
        "  print('validating')\n",
        "  preds = []\n",
        "  labels = []\n",
        "  # at the end of each training epoch, calculate the accuracy on the validation data\n",
        "  with torch.no_grad():\n",
        "    # set the model to evaluate mode so things like dropout are no longer random\n",
        "    model.eval()\n",
        "    valid_loss = []\n",
        "    for idx, batch in enumerate(valid_loader):\n",
        "      # send the batch to cuda\n",
        "      for k, v in batch.items():\n",
        "        batch[k] = v.to(torch.device('cuda'))\n",
        "      loss, pred = model(**batch)\n",
        "      valid_loss.append(loss)\n",
        "      if idx % 5 == 0:\n",
        "        print(f'Epoch: {epoch} valid loss: {torch.tensor(valid_loss[-5:]).mean()}')\n",
        "  print(f'average valid loss: {torch.tensor(valid_loss).mean()}')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0\n",
            "training_loss 2.2565858364105225\n",
            "validating\n",
            "Epoch: 0 valid loss: 2.022676706314087\n",
            "Epoch: 0 valid loss: 2.050161600112915\n",
            "Epoch: 0 valid loss: 2.109633684158325\n",
            "Epoch: 0 valid loss: 2.021721363067627\n",
            "Epoch: 0 valid loss: 1.707619309425354\n",
            "Epoch: 0 valid loss: 1.9760822057724\n",
            "Epoch: 0 valid loss: 1.8700177669525146\n",
            "average valid loss: 1.961233377456665\n",
            "Epoch: 1\n",
            "training_loss 1.9429348707199097\n",
            "validating\n",
            "Epoch: 1 valid loss: 2.0210466384887695\n",
            "Epoch: 1 valid loss: 1.7757422924041748\n",
            "Epoch: 1 valid loss: 1.9089982509613037\n",
            "Epoch: 1 valid loss: 1.9186903238296509\n",
            "Epoch: 1 valid loss: 1.7960214614868164\n",
            "Epoch: 1 valid loss: 1.920635461807251\n",
            "Epoch: 1 valid loss: 1.9651975631713867\n",
            "average valid loss: 1.8740062713623047\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "eh7bikvGtVqY"
      },
      "source": [
        "Verify that the language model has been further trained by examining predictions."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8rfyo5m3g-yj",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "67180cbb-07a5-4f27-c757-49ed6e95045b"
      },
      "source": [
        "from transformers import pipeline\n",
        "\n",
        "fill_mask = pipeline(\n",
        "    \"fill-mask\",\n",
        "    model=model,\n",
        "    tokenizer=tokenizer\n",
        ")\n",
        "model.to(torch.device('cpu'))\n",
        "fill_mask(f'The worker sprained his {tokenizer.mask_token}')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'score': 0.3544308841228485,\n",
              "  'sequence': '[CLS] the worker sprained his ankle [SEP]',\n",
              "  'token': 10792,\n",
              "  'token_str': 'ankle'},\n",
              " {'score': 0.1934514343738556,\n",
              "  'sequence': '[CLS] the worker sprained his knee [SEP]',\n",
              "  'token': 6181,\n",
              "  'token_str': 'knee'},\n",
              " {'score': 0.11106089502573013,\n",
              "  'sequence': '[CLS] the worker sprained his wrist [SEP]',\n",
              "  'token': 7223,\n",
              "  'token_str': 'wrist'},\n",
              " {'score': 0.07629260420799255,\n",
              "  'sequence': '[CLS] the worker sprained his back [SEP]',\n",
              "  'token': 2067,\n",
              "  'token_str': 'back'},\n",
              " {'score': 0.05465313047170639,\n",
              "  'sequence': '[CLS] the worker sprained his shoulder [SEP]',\n",
              "  'token': 3244,\n",
              "  'token_str': 'shoulder'}]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "397puSmFhmXP"
      },
      "source": [
        "### Save Pretrained Language Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6nTcj1AFhWv9"
      },
      "source": [
        "model.save_pretrained('msha_pretrain')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_wa_zFHY4DXW"
      },
      "source": [
        "# Train Model for Text Classification"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2eRD6IO5hsCU"
      },
      "source": [
        "## Load Pretrained Language Model for Text Classification\n",
        "To use our pretrained language model for text classification we simply need to swap out the last few layers and train those for our classification task. We can easily do this in transformers as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "L-Tni5QeWTLr",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "83618986-a1cb-4ac3-8ecf-437793b7933f"
      },
      "source": [
        "from transformers import AutoModelForSequenceClassification\n",
        "\n",
        "model = AutoModelForSequenceClassification.from_pretrained('msha_pretrain', num_labels=len(labeler.classes_))\n",
        "print(model)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at msha_pretrain were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at msha_pretrain and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "DistilBertForSequenceClassification(\n",
            "  (distilbert): DistilBertModel(\n",
            "    (embeddings): Embeddings(\n",
            "      (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
            "      (position_embeddings): Embedding(512, 768)\n",
            "      (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "      (dropout): Dropout(p=0.1, inplace=False)\n",
            "    )\n",
            "    (transformer): Transformer(\n",
            "      (layer): ModuleList(\n",
            "        (0): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (1): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (2): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (3): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (4): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "        (5): TransformerBlock(\n",
            "          (attention): MultiHeadSelfAttention(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "            (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
            "          )\n",
            "          (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "          (ffn): FFN(\n",
            "            (dropout): Dropout(p=0.1, inplace=False)\n",
            "            (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
            "            (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
            "          )\n",
            "          (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
            "        )\n",
            "      )\n",
            "    )\n",
            "  )\n",
            "  (pre_classifier): Linear(in_features=768, out_features=768, bias=True)\n",
            "  (classifier): Linear(in_features=768, out_features=46, bias=True)\n",
            "  (dropout): Dropout(p=0.2, inplace=False)\n",
            ")\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fpj4sqv1z3-K"
      },
      "source": [
        "## Prepare Data for Text Classification\n",
        "This requires only a slight modification to our preparations for language model pretraining, instead of using the language model collator to generate masked words for prediction we will instead using the \"part of body\" codes from the MSHA dataset as our targets."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nURqQD8D0j6b"
      },
      "source": [
        "train_dataset = CustomDataset(dataframe=df_train, tokenizer=tokenizer, max_len=200,\n",
        "                              input_field='NARRATIVE', target_field='PART_CODE')\n",
        "valid_dataset = CustomDataset(dataframe=df_valid, tokenizer=tokenizer, max_len=200,\n",
        "                              input_field='NARRATIVE', target_field='PART_CODE')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K4DrHNK52zTX"
      },
      "source": [
        "def collate_for_classification(sampled_rows):\n",
        "  keys = sampled_rows[0].keys()\n",
        "  batch = {key: [] for key in keys}\n",
        "  # assemble the rows into lists of tensors for each input\n",
        "  for row in sampled_rows:\n",
        "    for k, v in row.items():\n",
        "      batch[k].append(torch.tensor(v, dtype=torch.long))\n",
        "  # stack the list of tensors into one big tensor and move it to the GPU\n",
        "  for k, v in batch.items():\n",
        "    batch[k] = torch.stack(v).to(torch.device('cuda'))\n",
        "  return batch"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "AU0JOD8F2czN"
      },
      "source": [
        "train_loader = DataLoader(train_dataset, batch_size=16, \n",
        "                          sampler=RandomSampler(train_dataset), collate_fn=collate_for_classification)\n",
        "valid_loader = DataLoader(valid_dataset, batch_size=16, \n",
        "                          sampler=SequentialSampler(valid_dataset), collate_fn=collate_for_classification)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zEjMliucOvyA",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a661675d-3c91-4e21-d0a0-3d3e5e051ce3"
      },
      "source": [
        "train_loader.__iter__().__next__()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'attention_mask': tensor([[1, 1, 1,  ..., 0, 0, 0],\n",
              "         [1, 1, 1,  ..., 0, 0, 0],\n",
              "         [1, 1, 1,  ..., 0, 0, 0],\n",
              "         ...,\n",
              "         [1, 1, 1,  ..., 0, 0, 0],\n",
              "         [1, 1, 1,  ..., 0, 0, 0],\n",
              "         [1, 1, 1,  ..., 0, 0, 0]], device='cuda:0'),\n",
              " 'input_ids': tensor([[ 101, 8430, 8479,  ...,    0,    0,    0],\n",
              "         [ 101, 7904, 2001,  ...,    0,    0,    0],\n",
              "         [ 101, 4542, 1998,  ...,    0,    0,    0],\n",
              "         ...,\n",
              "         [ 101, 7904, 2001,  ...,    0,    0,    0],\n",
              "         [ 101, 7904, 2001,  ...,    0,    0,    0],\n",
              "         [ 101, 7904, 2018,  ...,    0,    0,    0]], device='cuda:0'),\n",
              " 'labels': tensor([16,  3, 35,  9, 37, 16, 16,  4, 31, 35, 11, 17,  8,  3, 29, 21],\n",
              "        device='cuda:0')}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 31
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mNS7PpAKx38_"
      },
      "source": [
        "## Training the Model with our own Training Loop"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uxesBThI5FQb"
      },
      "source": [
        "### Option 1: Create a custom training loop"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WzK0isjlOBoN",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "99216288-dcb2-4f81-ba6d-4e783a407074"
      },
      "source": [
        "import transformers\n",
        "from sklearn.metrics import accuracy_score, f1_score\n",
        "\n",
        "model = model.to(torch.device('cuda'))\n",
        "optimizer = transformers.AdamW(params=model.parameters(), lr=1e-4)\n",
        "\n",
        "for epoch in range(5):\n",
        "  print(f'Epoch: {epoch}')\n",
        "  training_loss = []\n",
        "  # set the model to training mode so things like dropout behave correctly\n",
        "  model.train()\n",
        "  for idx, batch in enumerate(train_loader):\n",
        "    # calculate the model loss and predictions on our training batch\n",
        "    loss, pred = output = model(**batch)\n",
        "    # calculate change in loss with respect to parameters (i.e. gradient)\n",
        "    loss.backward()\n",
        "    training_loss.append(loss)\n",
        "    # adjust the parameters in the direction that reduces loss as measured by the gradient\n",
        "    optimizer.step()\n",
        "    # zero out the gradient as we're now moving on to the next batch\n",
        "    optimizer.zero_grad()\n",
        "  print(f'training_loss {torch.tensor(training_loss).mean()}')\n",
        "  print('validating')\n",
        "  preds = []\n",
        "  labels = []\n",
        "  # at the end of each epoch, calculate the loss and accuracy on the validation data\n",
        "  with torch.no_grad():\n",
        "    # set the model to evaluate mode so things like dropout are no longer random\n",
        "    model.eval()\n",
        "    valid_loss = []\n",
        "    pred_codes = []\n",
        "    true_codes = []\n",
        "    for idx, batch in enumerate(valid_loader):\n",
        "      # send the batch to cuda\n",
        "      for k, v in batch.items():\n",
        "        batch[k] = v.to(torch.device('cuda'))\n",
        "      loss, pred = model(**batch)\n",
        "      valid_loss.append(loss)\n",
        "      true_codes.append(batch['labels'].cpu())\n",
        "      pred_codes.append(pred.cpu())\n",
        "      if idx % 5 == 0:\n",
        "        print(f'Epoch: {epoch} valid loss: {torch.tensor(valid_loss[-5:]).mean()}')\n",
        "  print(f'average valid loss: {torch.tensor(valid_loss).mean()}')\n",
        "  all_true = torch.cat(true_codes, dim=0)\n",
        "  all_pred = torch.cat(pred_codes, dim=0).argmax(dim=1)\n",
        "  acc = accuracy_score(all_true, all_pred)\n",
        "  print(f'valid accuracy: {acc}')\n",
        "\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "training_loss 2.5757479667663574\n",
            "validating\n",
            "Epoch: 0 valid loss: 1.5770251750946045\n",
            "Epoch: 0 valid loss: 1.302986979484558\n",
            "Epoch: 0 valid loss: 1.8998000621795654\n",
            "Epoch: 0 valid loss: 1.5334079265594482\n",
            "Epoch: 0 valid loss: 1.8177725076675415\n",
            "Epoch: 0 valid loss: 1.435440182685852\n",
            "Epoch: 0 valid loss: 1.7725677490234375\n",
            "Epoch: 0 valid loss: 1.8031257390975952\n",
            "Epoch: 0 valid loss: 1.478172779083252\n",
            "Epoch: 0 valid loss: 1.5638761520385742\n",
            "Epoch: 0 valid loss: 1.5756371021270752\n",
            "Epoch: 0 valid loss: 1.524620771408081\n",
            "Epoch: 0 valid loss: 1.6226778030395508\n",
            "average valid loss: 1.612066626548767\n",
            "valid accuracy: 0.646\n",
            "Epoch: 1\n",
            "training_loss 1.3201613426208496\n",
            "validating\n",
            "Epoch: 1 valid loss: 0.9834232926368713\n",
            "Epoch: 1 valid loss: 0.9454551935195923\n",
            "Epoch: 1 valid loss: 1.4750162363052368\n",
            "Epoch: 1 valid loss: 1.101564645767212\n",
            "Epoch: 1 valid loss: 1.3577848672866821\n",
            "Epoch: 1 valid loss: 1.0907435417175293\n",
            "Epoch: 1 valid loss: 1.207595944404602\n",
            "Epoch: 1 valid loss: 1.4173157215118408\n",
            "Epoch: 1 valid loss: 1.1742541790008545\n",
            "Epoch: 1 valid loss: 1.1272637844085693\n",
            "Epoch: 1 valid loss: 1.0069167613983154\n",
            "Epoch: 1 valid loss: 1.0839531421661377\n",
            "Epoch: 1 valid loss: 1.221519112586975\n",
            "average valid loss: 1.1953513622283936\n",
            "valid accuracy: 0.703\n",
            "Epoch: 2\n",
            "training_loss 0.8646737933158875\n",
            "validating\n",
            "Epoch: 2 valid loss: 0.7710725665092468\n",
            "Epoch: 2 valid loss: 0.7338489294052124\n",
            "Epoch: 2 valid loss: 1.4614112377166748\n",
            "Epoch: 2 valid loss: 0.9597587585449219\n",
            "Epoch: 2 valid loss: 1.130893349647522\n",
            "Epoch: 2 valid loss: 0.8599791526794434\n",
            "Epoch: 2 valid loss: 0.9734356999397278\n",
            "Epoch: 2 valid loss: 1.3107632398605347\n",
            "Epoch: 2 valid loss: 0.9585391283035278\n",
            "Epoch: 2 valid loss: 0.9222793579101562\n",
            "Epoch: 2 valid loss: 0.8695846796035767\n",
            "Epoch: 2 valid loss: 0.9008678197860718\n",
            "Epoch: 2 valid loss: 0.9125474691390991\n",
            "average valid loss: 1.0173168182373047\n",
            "valid accuracy: 0.741\n",
            "Epoch: 3\n",
            "training_loss 0.5844218730926514\n",
            "validating\n",
            "Epoch: 3 valid loss: 0.7816792130470276\n",
            "Epoch: 3 valid loss: 0.765034556388855\n",
            "Epoch: 3 valid loss: 1.663458228111267\n",
            "Epoch: 3 valid loss: 1.094512701034546\n",
            "Epoch: 3 valid loss: 1.17751944065094\n",
            "Epoch: 3 valid loss: 0.8817553520202637\n",
            "Epoch: 3 valid loss: 0.9378258585929871\n",
            "Epoch: 3 valid loss: 1.3505367040634155\n",
            "Epoch: 3 valid loss: 1.004450798034668\n",
            "Epoch: 3 valid loss: 0.8823951482772827\n",
            "Epoch: 3 valid loss: 0.8858342170715332\n",
            "Epoch: 3 valid loss: 0.9933502078056335\n",
            "Epoch: 3 valid loss: 0.9534024000167847\n",
            "average valid loss: 1.0657960176467896\n",
            "valid accuracy: 0.713\n",
            "Epoch: 4\n",
            "training_loss 0.3891642987728119\n",
            "validating\n",
            "Epoch: 4 valid loss: 0.7024602293968201\n",
            "Epoch: 4 valid loss: 0.7666038274765015\n",
            "Epoch: 4 valid loss: 1.456653356552124\n",
            "Epoch: 4 valid loss: 1.0137600898742676\n",
            "Epoch: 4 valid loss: 1.1287809610366821\n",
            "Epoch: 4 valid loss: 0.8076772689819336\n",
            "Epoch: 4 valid loss: 0.9413505792617798\n",
            "Epoch: 4 valid loss: 1.2599595785140991\n",
            "Epoch: 4 valid loss: 0.9287629127502441\n",
            "Epoch: 4 valid loss: 0.8950864672660828\n",
            "Epoch: 4 valid loss: 0.9324126243591309\n",
            "Epoch: 4 valid loss: 0.9428611993789673\n",
            "Epoch: 4 valid loss: 0.8803423643112183\n",
            "average valid loss: 1.0051395893096924\n",
            "valid accuracy: 0.747\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "waxVhiU510Xj"
      },
      "source": [
        "### Option 2: Use the Transformers Trainer\n",
        "It is easy to make mistakes when constructing the training loop by hand so the transformer's library also provides the Trainer class, which abstracts away the training, optimization, and validation. This makes it easier to train the model, but harder to debug or customize the training loop. \n",
        "\n",
        "By default the trainer expects the model to produce the loss as the first in a tuple of outputs when \"labels\" are provided as an input. This is already the default behavior of Transformers models so no additional modifications are necessary in our case."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0EHCUFINKcGZ"
      },
      "source": [
        "from transformers import Trainer, TrainingArguments\n",
        "from sklearn.metrics import accuracy_score\n",
        "\n",
        "def get_metrics(eval_prediction):\n",
        "  y_true = eval_prediction.label_ids\n",
        "  y_pred = torch.from_numpy(eval_prediction.predictions).softmax(-1).argmax(axis=1)\n",
        "  acc = accuracy_score(y_true, y_pred)\n",
        "  return {'accuracy': acc}\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='./results',          # output directory\n",
        "    num_train_epochs=3,              # total # of training epochs\n",
        "    per_device_train_batch_size=32,  # batch size per device during training\n",
        "    per_device_eval_batch_size=32,   # batch size for evaluation\n",
        "    warmup_steps=500,                # number of warmup steps for learning rate scheduler\n",
        "    weight_decay=0.01,               # strength of weight decay\n",
        "    logging_dir='./logs',            # directory for storing logs\n",
        "    save_total_limit=3,\n",
        "    do_eval=True,\n",
        "    evaluation_strategy='epoch',\n",
        "    learning_rate=1e-4\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,                         # the instantiated 🤗 Transformers model to be trained\n",
        "    args=training_args,                  # training arguments, defined above\n",
        "    train_dataset=train_dataset,         # training dataset\n",
        "    eval_dataset=valid_dataset,          # evaluation dataset\n",
        "    data_collator=collate_for_classification,\n",
        "    compute_metrics=get_metrics          # metrics that we want computed\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4pqfXz2kh5-V",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 233
        },
        "outputId": "df3e3db2-e5d1-4a80-ca5e-77e6a2cff980"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: UserWarning: To copy construct from a tensor, it is recommended to use sourceTensor.clone().detach() or sourceTensor.clone().detach().requires_grad_(True), rather than torch.tensor(sourceTensor).\n",
            "  import sys\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='96' max='96' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [96/96 01:14, Epoch 3/3]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.970974</td>\n",
              "      <td>0.758000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.958075</td>\n",
              "      <td>0.757000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>No log</td>\n",
              "      <td>0.959449</td>\n",
              "      <td>0.756000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=96, training_loss=0.22440765301386514)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KY2lBQz68Iky"
      },
      "source": [
        "## Saving and Reloading the Trained Model\n",
        "When using the trainer, the underlying model is attached to the trainer as an attribute. We can access and save it as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WD7ZfFlENAgX"
      },
      "source": [
        "torch.save(trainer.model, 'my_torch_model')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ft0v5EK08NOx"
      },
      "source": [
        "my_reloaded_model = torch.load('my_torch_model')"
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}