Pretrained_Transformers_(deberta-v2).ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Pretrained Transformers (intro)",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyNgTXmMTmVcRKnSGHG48DSJ",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "c0c02745dbeb452a82839512bc4bf70a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_8203f80f293340959105d5a21dbaf565",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_6ff796021eef45a8b5569758690cdae4",
              "IPY_MODEL_74d92e2d52914eb89841a7e10d274016"
            ]
          }
        },
        "8203f80f293340959105d5a21dbaf565": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6ff796021eef45a8b5569758690cdae4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_35f9060971234351b20fccc2f89a6b6a",
            "_dom_classes": [],
            "description": "Train: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 583,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 583,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_1f8c5a9265f54cdb85265409897e8c66"
          }
        },
        "74d92e2d52914eb89841a7e10d274016": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1a369762c96f4c4b89c2f1d6e3d676e1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 583/583 [03:55&lt;00:00,  2.48it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b0fa6ce83fcd452e850e35d0c61ec445"
          }
        },
        "35f9060971234351b20fccc2f89a6b6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "1f8c5a9265f54cdb85265409897e8c66": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1a369762c96f4c4b89c2f1d6e3d676e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b0fa6ce83fcd452e850e35d0c61ec445": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f0fb783e26cf4b198d1af991dec7ee69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_7631288e789549c0bab87b741389ec7f",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_25734f7622e649739ed1a74ab3a9db2e",
              "IPY_MODEL_78391301d4d6430586b72e7c5771bdde"
            ]
          }
        },
        "7631288e789549c0bab87b741389ec7f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "25734f7622e649739ed1a74ab3a9db2e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_de0c84ba90904b4ea15a22852902c6a4",
            "_dom_classes": [],
            "description": "Valid: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 142,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 142,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2cafd7c824f740d8b8b60dd07d79a72b"
          }
        },
        "78391301d4d6430586b72e7c5771bdde": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_986ebe3d834140efbf8e9d8491f07601",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 142/142 [02:33&lt;00:00,  1.08s/it]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5a2891aec8014704ae0cdc7eb2276084"
          }
        },
        "de0c84ba90904b4ea15a22852902c6a4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2cafd7c824f740d8b8b60dd07d79a72b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "986ebe3d834140efbf8e9d8491f07601": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5a2891aec8014704ae0cdc7eb2276084": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "75497f7a34f646d2923ff30651d671ea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c2ed3923807644659bc41fbbddb40b5c",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_240328cc3616415085f41e656a8499bb",
              "IPY_MODEL_4d10e1a6821a4d628dfe079b4d6e9598"
            ]
          }
        },
        "c2ed3923807644659bc41fbbddb40b5c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "240328cc3616415085f41e656a8499bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_098d821075e6430f909d408fc4ea153a",
            "_dom_classes": [],
            "description": "Train: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 583,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 583,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ca91c0f3d7f74977874100b13f110876"
          }
        },
        "4d10e1a6821a4d628dfe079b4d6e9598": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a199e744449e4461920cffde6aa596a1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 583/583 [02:18&lt;00:00,  4.22it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a8c40e0a3b69497bbd16ecb05aa2bbed"
          }
        },
        "098d821075e6430f909d408fc4ea153a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ca91c0f3d7f74977874100b13f110876": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a199e744449e4461920cffde6aa596a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a8c40e0a3b69497bbd16ecb05aa2bbed": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b5af59f196cd458dad9f18ffa1e851e1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c53d1cf0edd34568a88d9a15dcadb8e9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a16ed1300e4a4ce5b876650f8098bef4",
              "IPY_MODEL_b9cfafb378f249f0ae2572f4f8b7ca3c"
            ]
          }
        },
        "c53d1cf0edd34568a88d9a15dcadb8e9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a16ed1300e4a4ce5b876650f8098bef4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_85a80e863f804095b138cad55335c012",
            "_dom_classes": [],
            "description": "Valid: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 142,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 142,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8ec9e08fe89f47cbb74f49eedfd1fdb8"
          }
        },
        "b9cfafb378f249f0ae2572f4f8b7ca3c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1618fd5b75b24d849ae808016e7a03fb",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 142/142 [00:54&lt;00:00,  2.60it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_aaef61210931467ab498b819bfbb0c1b"
          }
        },
        "85a80e863f804095b138cad55335c012": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8ec9e08fe89f47cbb74f49eedfd1fdb8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1618fd5b75b24d849ae808016e7a03fb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "aaef61210931467ab498b819bfbb0c1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "31846d8ea097470ea360bad532bc0ba4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e8e823a10794405fa8d5a3c7d6336247",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0552bdf61d2a4c3ca10d0d6642877022",
              "IPY_MODEL_fe66bccbddd0419e9a8169d3ca5d381d"
            ]
          }
        },
        "e8e823a10794405fa8d5a3c7d6336247": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0552bdf61d2a4c3ca10d0d6642877022": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_c14581ff92284bb2b50097b63a154348",
            "_dom_classes": [],
            "description": "Train: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 583,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 583,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_08739b74eaa54d3795bfa5a125a50c68"
          }
        },
        "fe66bccbddd0419e9a8169d3ca5d381d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_ef98d65378a9473082c41c59b7dc1bef",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 583/583 [02:06&lt;00:00,  4.61it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_4e0d487cbccc4267bb72e053a0dc4529"
          }
        },
        "c14581ff92284bb2b50097b63a154348": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "08739b74eaa54d3795bfa5a125a50c68": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "ef98d65378a9473082c41c59b7dc1bef": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "4e0d487cbccc4267bb72e053a0dc4529": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8ae6420d154048aa8e7ea5ef93966155": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_2b734cf011364c2092f5043b81ee4846",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_aace88262c0b4b358f23c1abfb61ce5e",
              "IPY_MODEL_00d9d6b0264a462487a9b894009f622d"
            ]
          }
        },
        "2b734cf011364c2092f5043b81ee4846": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "aace88262c0b4b358f23c1abfb61ce5e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_603a0a1f590c478a87bf96f49396078a",
            "_dom_classes": [],
            "description": "Valid: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 142,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 142,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_ffe5501c1a5849f3843b3d432e8bfd06"
          }
        },
        "00d9d6b0264a462487a9b894009f622d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_e69b5ff2f193425998f6d57f1ef95f7f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 142/142 [00:42&lt;00:00,  3.34it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_f6fc40a7439544429362ce85538cbf71"
          }
        },
        "603a0a1f590c478a87bf96f49396078a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "ffe5501c1a5849f3843b3d432e8bfd06": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e69b5ff2f193425998f6d57f1ef95f7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "f6fc40a7439544429362ce85538cbf71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/ameasure/colab_tutorials/blob/master/Pretrained_Transformers_(deberta-v2).ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bIpB9LhIPGpH"
      },
      "source": [
        "# Text Classification with Transformers\n",
        "\n",
        "The most effective technique for most NLP tasks today is to take a transformer neural network, typically pretrained on massive collections of text data in an unsupervised manner, and then fine-tune it to your task. In this tutorial we'll show you how to use these techniques to train a text classifier for MSHA injury narratives.\n",
        "\n",
        "### Overview of the Transformer\n",
        "The transformer neural network was [introduced](https://arxiv.org/abs/1706.03762) in 2017 and originally designed to operate over a sequence of inputs representing words and/or subword sequences (i.e. tokens). The distinguishing operation of a transformer is self-attention. In self-attention, each input in a sequence is compared to every other input in the sequence and then aggregated to produce a new more context-aware sequence. Transformer networks typically consist of multiple blocks of self-attention, and incorporate numerous additional tricks to improve performance. Positional information about the sequence, which is necessary for many language tasks, is incorporated by adding a positional vector to each input in the sequence. \n",
        "\n",
        "A more extensive treatment of transformers is available [here](http://peterbloem.nl/blog/transformers).\n",
        "\n",
        "### Self-Supervision\n",
        "The second, far more important innovation we will use is self-supervised pretraining, which allows us to to initialize our model with language knowledge without access to labeled data. For language tasks, self-supervised pretraining is typically accomplished by gatherings huge collections of text, masking and/or corruption portions of the text and training the model to predict the missing or corrupted pieces. The first transformer to be pre-trained in this way is known as [BERT](https://arxiv.org/abs/1810.04805), and was introduced in 2018. Because it typically requires a huge amount of time to pre-train models on huge collections of text, we will not do that in this tutorial. Instead we will simply use one of the many that have already been pretrained for us.\n",
        "\n",
        "Our basic approach is as follows:\n",
        "1. Install the `transformers` library, which makes it easy to use pretrained transformers.\n",
        "2. Download our MSHA injury data for training and evaluation\n",
        "3. Load a pretrained transformer model with an untrained classification output layer for our task.\n",
        "4. Prepare our data so it's compatible with the inputs expected by the model.\n",
        "5. Finetune (i.e. further train) the pretrained transformer model on our MSHA data.\n",
        "6. Evaluate and use the model."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Lx4YYhzO6_8W"
      },
      "source": [
        "# Setup"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "2kaMTSnH7D3F"
      },
      "source": [
        "## Install Transformers Library"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IGYWeOkaO-_n"
      },
      "source": [
        "!pip install transformers>=4.4.2"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cHPZyxbZQOv4"
      },
      "source": [
        "## Download the MSHA Data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KHwxv-7DPFUa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "0c5b8533-98bb-4440-ca00-99f2d4fc62bf"
      },
      "source": [
        "!wget 'https://github.com/ameasure/autocoding-class/raw/master/msha.xlsx'"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "--2021-04-21 17:35:38--  https://github.com/ameasure/autocoding-class/raw/master/msha.xlsx\n",
            "Resolving github.com (github.com)... 140.82.112.4\n",
            "Connecting to github.com (github.com)|140.82.112.4|:443... connected.\n",
            "HTTP request sent, awaiting response... 302 Found\n",
            "Location: https://raw.githubusercontent.com/ameasure/autocoding-class/master/msha.xlsx [following]\n",
            "--2021-04-21 17:35:38--  https://raw.githubusercontent.com/ameasure/autocoding-class/master/msha.xlsx\n",
            "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.108.133, 185.199.111.133, 185.199.109.133, ...\n",
            "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.108.133|:443... connected.\n",
            "HTTP request sent, awaiting response... 200 OK\n",
            "Length: 4183086 (4.0M) [application/octet-stream]\n",
            "Saving to: ‘msha.xlsx’\n",
            "\n",
            "msha.xlsx           100%[===================>]   3.99M  --.-KB/s    in 0.04s   \n",
            "\n",
            "2021-04-21 17:35:39 (97.9 MB/s) - ‘msha.xlsx’ saved [4183086/4183086]\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ujGe33a0QVPV",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 255
        },
        "outputId": "e4b21833-34eb-4db7-e99b-99d9295eba18"
      },
      "source": [
        "import pandas as pd\n",
        "from sklearn import preprocessing\n",
        "\n",
        "df = pd.read_excel('msha.xlsx')\n",
        "df['ACCIDENT_YEAR'] = df['ACCIDENT_DT'].dt.year\n",
        "labeler = preprocessing.LabelEncoder()   # Labeler that will convert our codes to indexes\n",
        "labeler.fit(df['INJ_BODY_PART'])         # Calculate an index for each unique code\n",
        "df['PART_INDEX'] = labeler.transform(df['INJ_BODY_PART']) # Add the code indexes to our dataframe, we will need these later\n",
        "# separate training and validation by year\n",
        "df_train = df[df['ACCIDENT_YEAR'].isin([2010, 2011])].copy()\n",
        "df_valid = df[df['ACCIDENT_YEAR'] == 2012].copy()\n",
        "# show the results\n",
        "print('n_classes:', len(df['INJ_BODY_PART'].unique()))\n",
        "print('training rows:', len(df_train))\n",
        "print('validation rows:', len(df_valid))\n",
        "df[['INJ_BODY_PART', 'PART_INDEX', 'NARRATIVE', 'ACCIDENT_YEAR']].head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "n_classes: 46\n",
            "training rows: 18681\n",
            "validation rows: 9032\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>INJ_BODY_PART</th>\n",
              "      <th>PART_INDEX</th>\n",
              "      <th>NARRATIVE</th>\n",
              "      <th>ACCIDENT_YEAR</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)</td>\n",
              "      <td>35</td>\n",
              "      <td>Cleaning out Gabion Grizzly,  Rocks get Jammed...</td>\n",
              "      <td>2010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)</td>\n",
              "      <td>35</td>\n",
              "      <td>Injured was walking in the pit area, stepped o...</td>\n",
              "      <td>2010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>HIPS (PELVIS/ORGANS/KIDNEYS/BUTTOCKS)</td>\n",
              "      <td>22</td>\n",
              "      <td>Employee, parked s/c on grade at 16-Block #3 E...</td>\n",
              "      <td>2012</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>ANKLE</td>\n",
              "      <td>1</td>\n",
              "      <td>Contractor employee working as a carpenter mis...</td>\n",
              "      <td>2013</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>16</td>\n",
              "      <td>The employee's finger was pinched between the ...</td>\n",
              "      <td>2011</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                             INJ_BODY_PART  ...  ACCIDENT_YEAR\n",
              "0  SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)  ...           2010\n",
              "1  SHOULDERS (COLLARBONE/CLAVICLE/SCAPULA)  ...           2010\n",
              "2    HIPS (PELVIS/ORGANS/KIDNEYS/BUTTOCKS)  ...           2012\n",
              "3                                    ANKLE  ...           2013\n",
              "4                          FINGER(S)/THUMB  ...           2011\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vTYqVEF2Q4z9"
      },
      "source": [
        "## Load a Pretrained Transformer\n",
        "\n",
        "We will start with a pretrained transformer called `distilbert-base-uncased`, which is a carefully minitiarized version of BERT. \n",
        "\n",
        "In loading this model we are actually doing several things.\n",
        "1. We are loading the tokenizer used to train the transformer. This is important because it allows us to convert our narrative inputs into the format expected by the model.\n",
        "2. We are loading the model that was pretrained trained on data tokenized by this tokenizer.\n",
        "3. We are adapting the model to sequence classification, i.e. classification from a sequence of inputs. In practice this means chopping off the last part of the pretrained transformer and adding an untrained output layer (essentially a multinomial logistic regression), which will produce outputs for each of the codes we wish to assign. There are 46 part codes in our data, so we specify num_labels=46 when loading the model."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0bT9nFG-QY0n",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "758e6710-db70-424e-c3b1-7d2683658dfe"
      },
      "source": [
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import AutoModelForSequenceClassification, AutoTokenizer\n",
        "\n",
        "model_name = 'distilbert-base-uncased'\n",
        "model = AutoModelForSequenceClassification.from_pretrained(model_name, num_labels=46)\n",
        "tokenizer = AutoTokenizer.from_pretrained(model_name)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertForSequenceClassification: ['vocab_transform.weight', 'vocab_transform.bias', 'vocab_layer_norm.weight', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of DistilBertForSequenceClassification were not initialized from the model checkpoint at distilbert-base-uncased and are newly initialized: ['pre_classifier.weight', 'pre_classifier.bias', 'classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5AH5uRjFfOTl"
      },
      "source": [
        "# Preparing the Data\n",
        "\n",
        "The pretrained transformer expects inputs in a specific format (described in the [documentation](https://huggingface.co/transformers/model_doc/distilbert.html#distilbertforsequenceclassification)), the tokenizer is our primary mechanism for accomplishing this. We illustrate it's basic usage below:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "1bI8aAsrg3zt",
        "outputId": "5891bb49-418d-4f27-86d4-e932c486fe2d"
      },
      "source": [
        "inputs = tokenizer('Workers arm caught in grider', return_tensors='pt')\n",
        "print(inputs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': tensor([[ 101, 3667, 2849, 3236, 1999, 8370, 2121,  102]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1]])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jm-sEgQRhEkY"
      },
      "source": [
        "The tokenizer expects one or more input texts. In then converts them into two tensors, one consisting of indexes corresponding to the tokens that make up the word (`input_ids`), the other indicating the number of input words (`attention_mask`). These are the exact inputs expected by the model, which we can now invoke as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yrftDPPShc-E",
        "outputId": "13aa487e-09b9-4040-c398-3ef42468b699"
      },
      "source": [
        "output = model(**inputs)\n",
        "print(output)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SequenceClassifierOutput(loss=None, logits=tensor([[ 0.0749, -0.1781,  0.1764, -0.1064, -0.1286, -0.0361, -0.1055,  0.0268,\n",
            "          0.0013, -0.0534, -0.0041,  0.2025, -0.0888,  0.0731,  0.0904,  0.0598,\n",
            "          0.0416, -0.0253,  0.0262,  0.1578,  0.0420,  0.1467,  0.0392,  0.1198,\n",
            "          0.0584, -0.0358,  0.1646, -0.1632,  0.1824, -0.0613, -0.1047,  0.0661,\n",
            "         -0.0212,  0.0690, -0.0693, -0.0369, -0.0760,  0.0401,  0.0230, -0.0529,\n",
            "          0.0540,  0.0631,  0.0492,  0.0266,  0.0937,  0.1761]],\n",
            "       grad_fn=<AddmmBackward>), hidden_states=None, attentions=None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_UMu_N9Khj4s",
        "outputId": "551a1058-ba0f-4d4b-fa75-4e55d9e503cc"
      },
      "source": [
        "output['logits'].shape"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 46])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Za9x-UHrhtPP"
      },
      "source": [
        "The model output consists of 2 components:\n",
        "1. logits: [n_examples, n_labels] is the pre-softmax outputs from our final layer, one row for each input example, one column for each part of body classification\n",
        "2. loss: the model's loss on the inputs. Since no label was included in the inputs no loss was calculated. \n",
        "\n",
        "If we add the true labels to our inputs, the model will calculate the Negative Log Likelhood Loss (the pre-softmax equivalent of the Cross-Entropy Loss) for our model as shown below:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "io3aTyumiXxg",
        "outputId": "f9194ad4-02dc-4487-a5b4-c30ab5bb4715"
      },
      "source": [
        "inputs['labels'] = torch.tensor([[3]])\n",
        "outputs = model(**inputs)\n",
        "print(outputs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "SequenceClassifierOutput(loss=tensor(3.9611, grad_fn=<NllLossBackward>), logits=tensor([[ 0.0749, -0.1781,  0.1764, -0.1064, -0.1286, -0.0361, -0.1055,  0.0268,\n",
            "          0.0013, -0.0534, -0.0041,  0.2025, -0.0888,  0.0731,  0.0904,  0.0598,\n",
            "          0.0416, -0.0253,  0.0262,  0.1578,  0.0420,  0.1467,  0.0392,  0.1198,\n",
            "          0.0584, -0.0358,  0.1646, -0.1632,  0.1824, -0.0613, -0.1047,  0.0661,\n",
            "         -0.0212,  0.0690, -0.0693, -0.0369, -0.0760,  0.0401,  0.0230, -0.0529,\n",
            "          0.0540,  0.0631,  0.0492,  0.0266,  0.0937,  0.1761]],\n",
            "       grad_fn=<AddmmBackward>), hidden_states=None, attentions=None)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xPlfd6XHjJpV"
      },
      "source": [
        "The loss allows us to calculate the parameter gradients, which in turn tells us how to update parameter values when training the model with gradient descent.\n",
        "\n",
        "Probabilistic-like outputs from the final model can be recovered by applying the softmax to the logits:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KxoX_M4TjFmF",
        "outputId": "169446df-1baa-403c-fe62-0fbfb0a2c081"
      },
      "source": [
        "torch.softmax(outputs['logits'], -1)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "tensor([[0.0228, 0.0177, 0.0253, 0.0190, 0.0186, 0.0204, 0.0191, 0.0218, 0.0212,\n",
              "         0.0201, 0.0211, 0.0259, 0.0194, 0.0228, 0.0232, 0.0225, 0.0221, 0.0207,\n",
              "         0.0217, 0.0248, 0.0221, 0.0245, 0.0220, 0.0239, 0.0225, 0.0204, 0.0250,\n",
              "         0.0180, 0.0254, 0.0199, 0.0191, 0.0226, 0.0207, 0.0227, 0.0198, 0.0204,\n",
              "         0.0196, 0.0220, 0.0217, 0.0201, 0.0224, 0.0226, 0.0222, 0.0218, 0.0233,\n",
              "         0.0253]], grad_fn=<SoftmaxBackward>)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "_-RAv1hnk0Du"
      },
      "source": [
        "### About the Tokenizer\n",
        "\n",
        "You might have noticed the tokenizer is creating more tokens that we have input words. There are two reasons for this:\n",
        "1. Its adding special characters that are expected by the model.\n",
        "2. Its using a combination of word and subword tokens so it can represent all possible words with a very limited vocabularly.\n",
        "\n",
        "We can see both of these effects more clearly by partially reversing the tokenization below:\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8PJ7nnjuqMFs",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "4a52cec6-9185-4d88-b2ac-52adc7df88ea"
      },
      "source": [
        "inputs = tokenizer('Workers arm crushed by anthracite.')\n",
        "print(inputs)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': [101, 3667, 2849, 10560, 2011, 14405, 13492, 17847, 1012, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6rIuQ3YMltpR",
        "outputId": "ae00dc3a-2f16-4c8d-d445-d0c988de5f83"
      },
      "source": [
        "tokenizer.convert_ids_to_tokens(inputs['input_ids'])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[CLS]',\n",
              " 'workers',\n",
              " 'arm',\n",
              " 'crushed',\n",
              " 'by',\n",
              " 'ant',\n",
              " '##hra',\n",
              " '##cite',\n",
              " '.',\n",
              " '[SEP]']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 35
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ZZjfn6VHf1jT"
      },
      "source": [
        "Note the special `[CLS]` and `[SEP]` tokens, to indicate the start and end of the sequence, and the subword tokenization of `anthracite` into `ant`, `##hra`, `##cite`. "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uZkf1HNiVzP2"
      },
      "source": [
        "# Generating Batches\n",
        "\n",
        "Neural networks tend to be extraordinarily computationally expensive and that requires very careful use of our computing resources. One of the most important ways we accomplish this is computing on batches of data as it's rarely possible to compute on the entire dataset at once.\n",
        "\n",
        "To help us assemble these batches, Pytorch provides special Datasets, Samplers, Loaders, and collators. We describe each below."
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "In-0sOxsxxlR"
      },
      "source": [
        "## PyTorch Dataset\n",
        "The PyTorch Dataset is a representation of our data (training, validation, or test) that has a `__len__` method, so PyTorch knows how many examples it contains, and a `__get_item__` method so PyTorch can retrieve an example by specifying it's index (i.e. row number). \n",
        "\n",
        "We create our CustomDataset below by subclassing the Pytorch Dataset and modifying the associated methods to fit our purposes. In particular, we modify the `__get_item__` method so it produces a tokenized version of the corresponding narrative and the integer version of the code associated with that narrative if target_field is specified."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VVSSx5iBxv2C"
      },
      "source": [
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "\n",
        "class CustomDataset(Dataset):\n",
        "  def __init__(self, df, tokenizer, text_field, max_len=200, label_field=None):\n",
        "    self.df = df                    # dataframe containing our data\n",
        "    self.tokenizer = tokenizer      # pretrained tokenizer to convert narratives to indexes\n",
        "    self.text_field = text_field    # column of dataframe containing input text\n",
        "    self.max_len = max_len          # optional: max tokens we will use in a text\n",
        "    self.label_field = label_field  # optional: field containing example label\n",
        "\n",
        "  def __len__(self):\n",
        "    return len(self.df)\n",
        "\n",
        "  def __getitem__(self, index):\n",
        "    row = self.df.iloc[index]              # retrieve example at specified index\n",
        "    inputs = self.tokenizer.encode_plus(  \n",
        "                text=row[self.text_field], # specify the field to tokenize\n",
        "                max_length=self.max_len,   # set the max tokens we will allow in narrative (typically < 512 because of memory limitations)\n",
        "                truncation=True)           # truncate the narrative to max_len if it exceeds max_len (avoids memory errors)\n",
        "    if self.label_field:                   # if label_field is specified, we include labels in each example\n",
        "        inputs['labels'] = row[self.label_field]\n",
        "    return dict(inputs) "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1GoK5D56XnKk"
      },
      "source": [
        "Example of converting our df_train and df_valid dataframes into PyTorch Datasets."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qI4SO9inbw7B"
      },
      "source": [
        "train_dataset = CustomDataset(df=df_train, tokenizer=tokenizer, max_len=200,\n",
        "                              text_field='NARRATIVE', label_field='PART_INDEX')\n",
        "valid_dataset = CustomDataset(df=df_valid, tokenizer=tokenizer, max_len=200,\n",
        "                              text_field='NARRATIVE', label_field='PART_INDEX')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "poG1K6xx-HoC"
      },
      "source": [
        "Example of retrieving a row from our dataset by specifying the row index (in this case row 0)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Jh6IYjWHGOil",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "08b90222-0e66-4af0-bb38-633dea40fed0"
      },
      "source": [
        "print(train_dataset[0])"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': [101, 9344, 2041, 11721, 26282, 2078, 24665, 29266, 1010, 5749, 2131, 21601, 1998, 7861, 22086, 4402, 2038, 2000, 2593, 29198, 2030, 5245, 1996, 5749, 2041, 1997, 1996, 24665, 29266, 1012, 7904, 2001, 2478, 1037, 22889, 24225, 8691, 2000, 2131, 1996, 5749, 4558, 1998, 2766, 2242, 1999, 2010, 2157, 3244, 1012, 102], 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], 'labels': 35}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "K8brn_rR-Obm"
      },
      "source": [
        "## PyTorch Sampler\n",
        "\n",
        "The job of the PyTorch Sampler is to decide which examples need to be sampled from our dataset next. For training, we typically want to do this randomly without replacement so that each example is selected an equal number of times. We can accomplish this with the RandomSampler, illustrated below. "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5EtNllQQ8qoW",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "819a1757-1471-426b-afc9-9c1df21dd083"
      },
      "source": [
        "from torch.utils.data import RandomSampler\n",
        "\n",
        "random_sampler = RandomSampler(train_dataset)\n",
        "sampled_index = random_sampler.__iter__().__next__()\n",
        "print(sampled_index)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "14572\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y_Y3tZttgDU2"
      },
      "source": [
        "When training sequence models however, we can often get large speedups by sampling in such a way that each batch consists of narratives of similar length. The LengthGroupedSampler does exactly this so we will use it instead."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X9W6XxCZNzHz",
        "outputId": "ad63813c-6f47-44e9-88aa-dd49f8c42589"
      },
      "source": [
        "from transformers.trainer_pt_utils import LengthGroupedSampler\n",
        "\n",
        "length_sampler = LengthGroupedSampler(train_dataset, batch_size=32)\n",
        "length_sampler.__iter__().__next__()\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "6186"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "X8Wyf-N6fTqh"
      },
      "source": [
        "Once the model is trained, we typically only want to pull each example once, in order. This is accomplished using the SequentialSampler."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "pp3t7wm-fR9d",
        "outputId": "8d6e49a5-4e50-4994-e73a-77691390d5bd"
      },
      "source": [
        "from torch.utils.data import SequentialSampler\n",
        "\n",
        "sequential_sampler = SequentialSampler(train_dataset)\n",
        "sequential_sampler.__iter__().__next__()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vrUKFmkaYssi"
      },
      "source": [
        "## Collate Function\n",
        "\n",
        "Once the samples are drawn, we need to combine them into tensors of equal size. This is slightly complicated by the fact that different narratives are of different lengths. DataCollatorWithPadding solves this by padding each example in the batch to equal length using special padding tokens, and then combining the result into a tensor.   "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "T0VOAMyIbE-f"
      },
      "source": [
        "from transformers.data.data_collator import DataCollatorWithPadding\n",
        "\n",
        "pad_and_collate = DataCollatorWithPadding(tokenizer=tokenizer)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4FRzITd9Z5iJ"
      },
      "source": [
        "## DataLoader\n",
        "The job of the dataloader is connect the dataset, sampler, and collator so we can easily generate batches of the desired size. Specifically: the dataloader uses the sampler to retrieve examples from the dataset until it reaches the number specified by batch_size. It then uses the collate function to combine these into tensors. \n",
        "\n",
        "As a general rule we typically set the batch size to the largest number that can fit in our GPU's memory as larger batch sizes tend to enable faster training. If you set the batch size too high however you will get memory errors when training or using the model. If that happens you should restart the runtime and try a smaller batch size or a smaller max sequence length (transformer models in particular require large amounts of memory for long narratives)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "qelYKsus-YKw"
      },
      "source": [
        "from torch.utils.data import DataLoader\n",
        "\n",
        "train_loader = DataLoader(train_dataset, \n",
        "                          batch_size=32,                                              # number of training examples in each batch\n",
        "                          sampler=LengthGroupedSampler(train_dataset, batch_size=32), # sample batches with similar length\n",
        "                          collate_fn=pad_and_collate,\n",
        "                          pin_memory=True, # optimize memory transfer to from CPU to GPU\n",
        "                          drop_last=True)  # if last batch is smaller than batch_size, drop it (usually best for training)\n",
        "\n",
        "valid_loader = DataLoader(valid_dataset, \n",
        "                          batch_size=64,                                              # number of validation examples in each batch (can usually be double training)\n",
        "                          sampler=LengthGroupedSampler(valid_dataset, batch_size=64), # sample batches with similar length\n",
        "                          collate_fn=pad_and_collate,\n",
        "                          pin_memory=True, # optimize memory transfer from CPU to GPU\n",
        "                          drop_last=False) # don't drop partial batches (best for evaluation and inference)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SSIetN7yLZ4L"
      },
      "source": [
        "Example of pulling a batch from our dataloader"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wfn5cOCVLYjV",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "8863e45e-e93a-42dd-bcc4-aea962e39735"
      },
      "source": [
        "batch = train_loader.__iter__().__next__()\n",
        "print(batch)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "{'input_ids': tensor([[  101,  2012,  2260,  ...,  1015,  1012,   102],\n",
            "        [  101, 25212,  2187,  ...,     0,     0,     0],\n",
            "        [  101,  3626,  2001,  ...,     0,     0,     0],\n",
            "        ...,\n",
            "        [  101, 25212,  2001,  ...,     0,     0,     0],\n",
            "        [  101,  2006,  1017,  ...,     0,     0,     0],\n",
            "        [  101,  1996,  5043,  ...,     0,     0,     0]]), 'attention_mask': tensor([[1, 1, 1,  ..., 1, 1, 1],\n",
            "        [1, 1, 1,  ..., 0, 0, 0],\n",
            "        [1, 1, 1,  ..., 0, 0, 0],\n",
            "        ...,\n",
            "        [1, 1, 1,  ..., 0, 0, 0],\n",
            "        [1, 1, 1,  ..., 0, 0, 0],\n",
            "        [1, 1, 1,  ..., 0, 0, 0]]), 'labels': tensor([ 6,  4,  1,  4, 32,  3,  4, 12, 15,  9, 35,  4, 24, 21, 31,  4, 17, 30,\n",
            "        17, 31, 31, 45, 42, 31, 35, 26,  3,  4,  7, 16, 24, 15])}\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4HP5NLa8lzAt"
      },
      "source": [
        "Note that the input_ids and attention_mask tensors have been padded to equal size (the pad_token is 0)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "hHK8fmbpbi1M",
        "outputId": "0b866517-52f0-4a4c-a044-821e198f263a"
      },
      "source": [
        "batch['input_ids'].size()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([32, 137])"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 16
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "DhEcwZt4LicB"
      },
      "source": [
        "# Train the Model\n",
        "\n",
        "The standard approach to training a neural network is to use batch gradient descent. Specifically, over a series of epochs (i.e. passes through the training dataset) we will repeatedly:\n",
        "1. Retrieve a batch of training examples (by iterating through the train_loader)\n",
        "2. Calculate the model's predictions ('logits') and ('loss') on the batch\n",
        "3. Calculate the gradient, i.e. how much of the loss can be attributed to each of the model parameters\n",
        "4. Update the model parameters in the direction that reduces loss, as measured by the gradient\n",
        "5. Repeat until we reach a stopping criteria, such as validation performance no longer improving or a specified number of epochs being reached.\n",
        "\n",
        "To monitor performance, at the end of each training epoch we will:\n",
        "1. Retrieve batches from the validation data\n",
        "2. Calculate the model's predictions on these batches\n",
        "3. Compare these predictions to the labels to calculate the accuracy and f1-score\n",
        "This helps us determine how many training epochs we should use (if validation accuracy and macro-f1 stop improving, it's time to stop)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IjEbd57TC0Eq",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 482,
          "referenced_widgets": [
            "c0c02745dbeb452a82839512bc4bf70a",
            "8203f80f293340959105d5a21dbaf565",
            "6ff796021eef45a8b5569758690cdae4",
            "74d92e2d52914eb89841a7e10d274016",
            "35f9060971234351b20fccc2f89a6b6a",
            "1f8c5a9265f54cdb85265409897e8c66",
            "1a369762c96f4c4b89c2f1d6e3d676e1",
            "b0fa6ce83fcd452e850e35d0c61ec445",
            "f0fb783e26cf4b198d1af991dec7ee69",
            "7631288e789549c0bab87b741389ec7f",
            "25734f7622e649739ed1a74ab3a9db2e",
            "78391301d4d6430586b72e7c5771bdde",
            "de0c84ba90904b4ea15a22852902c6a4",
            "2cafd7c824f740d8b8b60dd07d79a72b",
            "986ebe3d834140efbf8e9d8491f07601",
            "5a2891aec8014704ae0cdc7eb2276084",
            "75497f7a34f646d2923ff30651d671ea",
            "c2ed3923807644659bc41fbbddb40b5c",
            "240328cc3616415085f41e656a8499bb",
            "4d10e1a6821a4d628dfe079b4d6e9598",
            "098d821075e6430f909d408fc4ea153a",
            "ca91c0f3d7f74977874100b13f110876",
            "a199e744449e4461920cffde6aa596a1",
            "a8c40e0a3b69497bbd16ecb05aa2bbed",
            "b5af59f196cd458dad9f18ffa1e851e1",
            "c53d1cf0edd34568a88d9a15dcadb8e9",
            "a16ed1300e4a4ce5b876650f8098bef4",
            "b9cfafb378f249f0ae2572f4f8b7ca3c",
            "85a80e863f804095b138cad55335c012",
            "8ec9e08fe89f47cbb74f49eedfd1fdb8",
            "1618fd5b75b24d849ae808016e7a03fb",
            "aaef61210931467ab498b819bfbb0c1b",
            "31846d8ea097470ea360bad532bc0ba4",
            "e8e823a10794405fa8d5a3c7d6336247",
            "0552bdf61d2a4c3ca10d0d6642877022",
            "fe66bccbddd0419e9a8169d3ca5d381d",
            "c14581ff92284bb2b50097b63a154348",
            "08739b74eaa54d3795bfa5a125a50c68",
            "ef98d65378a9473082c41c59b7dc1bef",
            "4e0d487cbccc4267bb72e053a0dc4529",
            "8ae6420d154048aa8e7ea5ef93966155",
            "2b734cf011364c2092f5043b81ee4846",
            "aace88262c0b4b358f23c1abfb61ce5e",
            "00d9d6b0264a462487a9b894009f622d",
            "603a0a1f590c478a87bf96f49396078a",
            "ffe5501c1a5849f3843b3d432e8bfd06",
            "e69b5ff2f193425998f6d57f1ef95f7f",
            "f6fc40a7439544429362ce85538cbf71"
          ]
        },
        "outputId": "77f5212a-083f-418c-8907-85b7a8968641"
      },
      "source": [
        "from tqdm.notebook import tqdm # provides graphical update on training progress\n",
        "import transformers\n",
        "from transformers import AdamW\n",
        "from sklearn.metrics import accuracy_score, f1_score\n",
        "\n",
        "model = model.to(torch.device('cuda'))                 # transfer model parameters to GPU\n",
        "optimizer = AdamW(params=model.parameters(), lr=1e-4)  # set optimizer and learning rate (will update model parameters)\n",
        "\n",
        "for epoch in range(3):         # repeat 3 times\n",
        "  print(f'Epoch: {epoch}')\n",
        "  model.train()                # set dropout and similar layers to random mode\n",
        "  for batch in tqdm(train_loader, desc='Train'):    # iterate through all the batches in training data\n",
        "    batch = {k: v.cuda() for k, v in batch.items()} # transfer the batch to the GPU\n",
        "    output = model(**batch)    # calculate logits (predictions) and loss on training batch\n",
        "    output['loss'].backward()  # calculate gradient (change in loss with respect to parameters)\n",
        "    optimizer.step()           # adjust the parameters in the direction that reduces loss as measured by gradient\n",
        "    optimizer.zero_grad()      # zero out the gradient as we're now moving on to the next batch\n",
        "  # at the end of each epoch, evaluate our model on the validation data\n",
        "  preds = []                   # will accumulate validation predictions\n",
        "  labels = []                  # will accumulate validation true labels\n",
        "  with torch.no_grad():            # disable gradient tracking, its slow and not needed for validation\n",
        "    model.eval()                   # change dropout and similar layers to deterministic mode   \n",
        "    for batch in tqdm(valid_loader, desc='Valid'):    # iterate through batches of validation data\n",
        "      batch = {k: v.cuda() for k, v in batch.items()} # transfer the batch to the GPU\n",
        "      output = model(**batch)      # generate predictions on validation data batch [n_examples, n_labels]\n",
        "      labels.append(batch['labels'].cpu())     # transfer labels to CPU and append to list\n",
        "      preds.append(output['logits'].cpu())     # transfer logits to CPU and append to list\n",
        "  all_preds = torch.cat((preds), 0).argmax(-1) # stack predicted logits and retrieve best predictions\n",
        "  all_labels = torch.cat((labels), 0)          # stack true labels\n",
        "  acc = accuracy_score(all_labels, all_preds)            # calculate validation accuracy\n",
        "  mf1 = f1_score(all_labels, all_preds, average='macro') # calculate validation macro-f1\n",
        "  print(f'accuracy: {round(acc, 3)}') \n",
        "  print(f'micro-f1: {round(mf1, 3)}')\n"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Epoch: 0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "c0c02745dbeb452a82839512bc4bf70a",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Train', max=583.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "f0fb783e26cf4b198d1af991dec7ee69",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Valid', max=142.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "accuracy: 0.806\n",
            "micro-f1: 0.607\n",
            "Epoch: 1\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "75497f7a34f646d2923ff30651d671ea",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Train', max=583.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b5af59f196cd458dad9f18ffa1e851e1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Valid', max=142.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "accuracy: 0.808\n",
            "micro-f1: 0.61\n",
            "Epoch: 2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "31846d8ea097470ea360bad532bc0ba4",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Train', max=583.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "8ae6420d154048aa8e7ea5ef93966155",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Valid', max=142.0, style=ProgressStyle(description_width=…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "accuracy: 0.811\n",
            "micro-f1: 0.633\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "iJGWYcw9Abfc"
      },
      "source": [
        "## Saving the Model\n",
        "\n",
        "Training neural networks can take a very long time so we often want to save the resulting model. We can save it and reload it as follows."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mtukf1SKAisR"
      },
      "source": [
        "# save the model\n",
        "torch.save(model, 'path_for_my_trained_model')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vnhfCLL05Wvx"
      },
      "source": [
        "## Reloading the Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SVkJUzaP6kP3"
      },
      "source": [
        "# reload the model\n",
        "reloaded_model = torch.load('path_for_my_trained_model')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1SGYVBUGjt_9"
      },
      "source": [
        "## Using the Model\n",
        "\n",
        "Using the model to generate predictions is similar to using the model for validation. We will put the data into a PyTorch Dataset and DataLoader, and then retrieve batches, calculate predictions and probabilities on those batches, and finally save the results to a dataframe."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_Zm74ogfnXTT"
      },
      "source": [
        "pred_loader = DataLoader(valid_dataset, \n",
        "                         batch_size=64,                            # number of validation examples in each batch (can usually be double training)\n",
        "                         sampler=SequentialSampler(valid_dataset), # sample batches in order\n",
        "                         collate_fn=pad_and_collate,\n",
        "                         pin_memory=True, # optimize memory transfromer from CPU to GPU\n",
        "                         drop_last=False) # don't drop partial batches (we want predictions for all examples)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "f1GeIioX6p9R"
      },
      "source": [
        "preds = []                    # list that will accumulate all our batch predictions\n",
        "with torch.no_grad():         # disable gradient tracking, it's expensive and only useful for training\n",
        "  reloaded_model.eval()       # set model to evaluation mode so dropout is deterministic\n",
        "  for batch in pred_loader:   # iterate through sequential batches of validation data\n",
        "    batch = {k: v.cuda() for k, v in batch.items()} # transfer the batch to the GPU\n",
        "    output = model(**batch)   # generate model predictions on batch\n",
        "    preds.append(output['logits'].cpu())   # append the logits to preds\n",
        "  all_preds = torch.cat(preds, dim=0)      # concatenate our list of preds into one big array [n_valid, n_labels]\n",
        "  all_probs = torch.softmax(all_preds, -1) # calculate the probabilities associated with our logits [n_valid, n_labels]\n",
        "best_preds = all_probs.argmax(dim=1)       # calculate the highest probability prediction code index for each example\n",
        "best_pred_probs = all_probs.max(dim=1)[0]  # calculate the probability for each prediction"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xd51bnA97xN3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 348
        },
        "outputId": "f4fd41ea-5885-490a-920e-daae0d2dc100"
      },
      "source": [
        "# convert our predictions, which are code indexes, back to their string values\n",
        "df_valid['PRED_CODE'] = labeler.inverse_transform(best_preds)\n",
        "df_valid['PRED_PROB'] = best_pred_probs\n",
        "df_valid[['NARRATIVE', 'INJ_BODY_PART', 'PRED_CODE', 'PRED_PROB']].sample(10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>NARRATIVE</th>\n",
              "      <th>INJ_BODY_PART</th>\n",
              "      <th>PRED_CODE</th>\n",
              "      <th>PRED_PROB</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>547</th>\n",
              "      <td>The splitter operator pushed stone through jaw...</td>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>0.981081</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>36888</th>\n",
              "      <td>EE was cleaning out tracks on 230 excavator Jo...</td>\n",
              "      <td>KNEE/PATELLA</td>\n",
              "      <td>KNEE/PATELLA</td>\n",
              "      <td>0.992641</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>23870</th>\n",
              "      <td>DEHYDRATION</td>\n",
              "      <td>BODY SYSTEMS</td>\n",
              "      <td>BODY SYSTEMS</td>\n",
              "      <td>0.962100</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2144</th>\n",
              "      <td>Late afternoon, cloudy, misting rain.</td>\n",
              "      <td>WRIST</td>\n",
              "      <td>NECK</td>\n",
              "      <td>0.213622</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>814</th>\n",
              "      <td>Changing hose on double boom bolter and, when ...</td>\n",
              "      <td>HAND (NOT WRIST OR FINGERS)</td>\n",
              "      <td>HAND (NOT WRIST OR FINGERS)</td>\n",
              "      <td>0.975269</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>475</th>\n",
              "      <td>Employee states that while being loaded by 106...</td>\n",
              "      <td>NECK</td>\n",
              "      <td>NECK</td>\n",
              "      <td>0.972752</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>10572</th>\n",
              "      <td>Employee states that he injured his back while...</td>\n",
              "      <td>BACK (MUSCLES/SPINE/S-CORD/TAILBONE)</td>\n",
              "      <td>BACK (MUSCLES/SPINE/S-CORD/TAILBONE)</td>\n",
              "      <td>0.986079</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>31259</th>\n",
              "      <td>While attempting to cut a zip tie, safety lock...</td>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>0.661779</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>21391</th>\n",
              "      <td>The employee was installing roof screen while ...</td>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>FINGER(S)/THUMB</td>\n",
              "      <td>0.984873</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>37041</th>\n",
              "      <td>Ind. was clawing when he felt alot of pain in ...</td>\n",
              "      <td>KNEE/PATELLA</td>\n",
              "      <td>KNEE/PATELLA</td>\n",
              "      <td>0.989116</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                               NARRATIVE  ... PRED_PROB\n",
              "547    The splitter operator pushed stone through jaw...  ...  0.981081\n",
              "36888  EE was cleaning out tracks on 230 excavator Jo...  ...  0.992641\n",
              "23870                                        DEHYDRATION  ...  0.962100\n",
              "2144               Late afternoon, cloudy, misting rain.  ...  0.213622\n",
              "814    Changing hose on double boom bolter and, when ...  ...  0.975269\n",
              "475    Employee states that while being loaded by 106...  ...  0.972752\n",
              "10572  Employee states that he injured his back while...  ...  0.986079\n",
              "31259  While attempting to cut a zip tie, safety lock...  ...  0.661779\n",
              "21391  The employee was installing roof screen while ...  ...  0.984873\n",
              "37041  Ind. was clawing when he felt alot of pain in ...  ...  0.989116\n",
              "\n",
              "[10 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "waxVhiU510Xj"
      },
      "source": [
        "# Training the Model using the Trainer (optional)\n",
        "It is easy to make mistakes when constructing the training loop by hand, so the transformers library also provides the Trainer class, which abstracts away the training, optimization, and validation."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0EHCUFINKcGZ"
      },
      "source": [
        "from transformers import Trainer, TrainingArguments\n",
        "from sklearn.metrics import accuracy_score, f1_score\n",
        "\n",
        "def get_metrics(eval_prediction):\n",
        "  \"\"\" \n",
        "  Function specifying the metrics to periodically calculate on validation data \n",
        "  Input:\n",
        "    eval_prediction: evaluation object with two attributes\n",
        "      label_ids: the true labels\n",
        "      predictions: the model predictions\n",
        "  Output:\n",
        "    Dict[metric_name, metric_value]\n",
        "  \"\"\"\n",
        "  y_true = eval_prediction.label_ids\n",
        "  y_pred = torch.from_numpy(eval_prediction.predictions).softmax(-1).argmax(axis=1)\n",
        "  acc = accuracy_score(y_true, y_pred)\n",
        "  mf1 = f1_score(y_true, y_pred, average='macro')\n",
        "  return {'accuracy': acc,\n",
        "          'macro-f1': mf1}\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='./results',          # output directory\n",
        "    num_train_epochs=2,              # total # of training epochs (i.e. iterations through training dataset)\n",
        "    per_device_train_batch_size=32,  # batch size per device during training\n",
        "    per_device_eval_batch_size=64,   # batch size for evaluation (can usually be about 2x train_batch_size because no gradients are calculated)\n",
        "    warmup_steps=500,                # number of warmup steps for learning rate scheduler (often useful to gradually increase lr for transformers)\n",
        "    weight_decay=0.01,               # strength of weight decay (L2 regularization)\n",
        "    logging_dir='./logs',            # directory where logs are stored\n",
        "    save_strategy='epoch',           # when to save checkpoints of our model (epoch means at the end of every epoch)\n",
        "    save_total_limit=3,              # the maximum number of model checkpoints to save (by default they are saved every 500 steps)\n",
        "    do_eval=True,                    # whether to periodically evaluate on the validtion data\n",
        "    evaluation_strategy='epoch',     # how often we will evaluate (epoch means at the end of each epoch)\n",
        "    group_by_length=True,            # use the length sampler for training\n",
        "    learning_rate=1e-4               # the optimizer's learning rate, i.e. how much we adjust the weights at each step\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,                         # the model to be trained\n",
        "    args=training_args,                  # training arguments, defined above\n",
        "    train_dataset=train_dataset,         # training dataset\n",
        "    eval_dataset=valid_dataset,          # evaluation dataset\n",
        "    data_collator=DataCollatorWithPadding(tokenizer),\n",
        "    compute_metrics=get_metrics          # metrics that we want computed\n",
        ")"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4pqfXz2kh5-V",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 172
        },
        "outputId": "fd705169-4ca3-4300-eb7a-4a8595da8700"
      },
      "source": [
        "trainer.train()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='1168' max='1168' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [1168/1168 04:36, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Macro-f1</th>\n",
              "      <th>Runtime</th>\n",
              "      <th>Samples Per Second</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>0.415900</td>\n",
              "      <td>0.757202</td>\n",
              "      <td>0.808791</td>\n",
              "      <td>0.620547</td>\n",
              "      <td>24.992400</td>\n",
              "      <td>361.390000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.348100</td>\n",
              "      <td>0.779492</td>\n",
              "      <td>0.818313</td>\n",
              "      <td>0.626997</td>\n",
              "      <td>24.648800</td>\n",
              "      <td>366.428000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=1168, training_loss=0.36934691912507356, metrics={'train_runtime': 276.8585, 'train_samples_per_second': 4.219, 'total_flos': 677350883466708.0, 'epoch': 2.0, 'init_mem_cpu_alloc_delta': 0, 'init_mem_gpu_alloc_delta': 0, 'init_mem_cpu_peaked_delta': 0, 'init_mem_gpu_peaked_delta': 0, 'train_mem_cpu_alloc_delta': 2490368, 'train_mem_gpu_alloc_delta': 548987904, 'train_mem_cpu_peaked_delta': 0, 'train_mem_gpu_peaked_delta': 1982753280})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "CWG53Y3_zw2F"
      },
      "source": [
        "## Evaluate the Model Using the Trainer\n",
        "Evaluation happens on the eval (validation) data."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yr1UqL6EHtP_",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 219
        },
        "outputId": "e960f869-aa42-4aca-937e-ce2962fdb33e"
      },
      "source": [
        "trainer.evaluate()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='142' max='142' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [142/142 00:24]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'epoch': 2.0,\n",
              " 'eval_accuracy': 0.8183126660761736,\n",
              " 'eval_loss': 0.7794921398162842,\n",
              " 'eval_macro-f1': 0.6269974132894985,\n",
              " 'eval_mem_cpu_alloc_delta': 0,\n",
              " 'eval_mem_cpu_peaked_delta': 0,\n",
              " 'eval_mem_gpu_alloc_delta': 0,\n",
              " 'eval_mem_gpu_peaked_delta': 262299648,\n",
              " 'eval_runtime': 24.5903,\n",
              " 'eval_samples_per_second': 367.299}"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 25
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "KY2lBQz68Iky"
      },
      "source": [
        "## Saving and Reloading the Trainer Model\n",
        "When using the trainer, the underlying model is attached to the trainer as an attribute. We can access and save it as follows:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WD7ZfFlENAgX"
      },
      "source": [
        "torch.save(trainer.model, 'my_torch_model')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Ft0v5EK08NOx"
      },
      "source": [
        "my_reloaded_model = torch.load('my_torch_model')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SAURDIheMUEF"
      },
      "source": [
        "## Resuming Training\n",
        "Alternately, you can resume training from a previous checkpoint by specifying the checkpoint in the train method.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 172
        },
        "id": "K-oGFfX6MfpP",
        "outputId": "9a3c668f-736b-4353-c7b7-7c5d2dab6c39"
      },
      "source": [
        "training_args.num_train_epochs += 1      # add one more training epoch\n",
        "\n",
        "trainer = Trainer(\n",
        "    model=model,                         # the model to be trained\n",
        "    args=training_args,                  # training arguments, defined above\n",
        "    train_dataset=train_dataset,         # training dataset\n",
        "    eval_dataset=valid_dataset,          # evaluation dataset\n",
        "    data_collator=DataCollatorWithPadding(tokenizer),\n",
        "    compute_metrics=get_metrics          # metrics that we want computed\n",
        ")\n",
        "\n",
        "trainer.train(r'./results/checkpoint-1168')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "        <style>\n",
              "            /* Turns off some styling */\n",
              "            progress {\n",
              "                /* gets rid of default border in Firefox and Opera. */\n",
              "                border: none;\n",
              "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
              "                background-size: auto;\n",
              "            }\n",
              "        </style>\n",
              "      \n",
              "      <progress value='2336' max='2336' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [2336/2336 05:19, Epoch 4/4]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "      <th>Accuracy</th>\n",
              "      <th>Macro-f1</th>\n",
              "      <th>Runtime</th>\n",
              "      <th>Samples Per Second</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>3</td>\n",
              "      <td>0.225600</td>\n",
              "      <td>0.886861</td>\n",
              "      <td>0.812223</td>\n",
              "      <td>0.650700</td>\n",
              "      <td>24.924000</td>\n",
              "      <td>362.382000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>4</td>\n",
              "      <td>0.178800</td>\n",
              "      <td>0.951256</td>\n",
              "      <td>0.813330</td>\n",
              "      <td>0.628196</td>\n",
              "      <td>24.786900</td>\n",
              "      <td>364.386000</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=2336, training_loss=0.08691935179984733, metrics={'train_runtime': 319.8689, 'train_samples_per_second': 7.303, 'total_flos': 1355118973466304.0, 'epoch': 4.0, 'init_mem_cpu_alloc_delta': 0, 'init_mem_gpu_alloc_delta': 0, 'init_mem_cpu_peaked_delta': 0, 'init_mem_gpu_peaked_delta': 0, 'train_mem_cpu_alloc_delta': -10366976, 'train_mem_gpu_alloc_delta': 1083420672, 'train_mem_cpu_peaked_delta': 13570048, 'train_mem_gpu_peaked_delta': 1982733824})"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 33
        }
      ]
    }
  ]
}