Visualizing_gpt2_token_embeddings.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "authorship_tag": "ABX9TyPYa3wZxis7KQY6gALQL+N8",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "faaf0e5fd82d43c58863f6bdf0f7e838": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aae64e9205094a1399d0e6eb19c2b3f9",
              "IPY_MODEL_cde9e650e29a44e2a0a535ebf00e4295",
              "IPY_MODEL_409497a680af41a59e4a322b3fe026b8"
            ],
            "layout": "IPY_MODEL_a26faaf0e540456a957a72d4a2985adc"
          }
        },
        "aae64e9205094a1399d0e6eb19c2b3f9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e2290dc6664d480686a484da82ee5b36",
            "placeholder": "​",
            "style": "IPY_MODEL_39b3bd2d816a4c49a03d8f56e6cdabee",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "cde9e650e29a44e2a0a535ebf00e4295": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5151b3419fb48a1b638a92d42c9d186",
            "max": 665,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_f372db21eefa4e648da1acb00cbcbad0",
            "value": 665
          }
        },
        "409497a680af41a59e4a322b3fe026b8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bb880391a9014cf7ba71d6d4b178b030",
            "placeholder": "​",
            "style": "IPY_MODEL_06fa0079c51e4f6b85a505944c4734ac",
            "value": " 665/665 [00:00&lt;00:00, 14.1kB/s]"
          }
        },
        "a26faaf0e540456a957a72d4a2985adc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e2290dc6664d480686a484da82ee5b36": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "39b3bd2d816a4c49a03d8f56e6cdabee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5151b3419fb48a1b638a92d42c9d186": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f372db21eefa4e648da1acb00cbcbad0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "bb880391a9014cf7ba71d6d4b178b030": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06fa0079c51e4f6b85a505944c4734ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "73a0640d1378478180828723624b4f24": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_aea2750dd1bd43058c503b4b071c5a96",
              "IPY_MODEL_f6e77fcc8854431a981f32f2001155d6",
              "IPY_MODEL_c99b7190a0df4bd195d1cbc7f0d1b5ac"
            ],
            "layout": "IPY_MODEL_fa60531273324039b35de8f6e1547065"
          }
        },
        "aea2750dd1bd43058c503b4b071c5a96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_830603676fcc489183b38bcb9b609a50",
            "placeholder": "​",
            "style": "IPY_MODEL_eed8fc183c6b465789682174ee41c6d1",
            "value": "Downloading pytorch_model.bin: 100%"
          }
        },
        "f6e77fcc8854431a981f32f2001155d6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9b76654088384d99af7e9303fe0ee877",
            "max": 548118077,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_c119b997fc4844f0acadd691089ccee5",
            "value": 548118077
          }
        },
        "c99b7190a0df4bd195d1cbc7f0d1b5ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3634e4dad04e424faa63d5056bba3f16",
            "placeholder": "​",
            "style": "IPY_MODEL_8ed1f250917c4e70ba70b2fb4afd9a12",
            "value": " 548M/548M [00:06&lt;00:00, 96.6MB/s]"
          }
        },
        "fa60531273324039b35de8f6e1547065": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "830603676fcc489183b38bcb9b609a50": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eed8fc183c6b465789682174ee41c6d1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9b76654088384d99af7e9303fe0ee877": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c119b997fc4844f0acadd691089ccee5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "3634e4dad04e424faa63d5056bba3f16": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8ed1f250917c4e70ba70b2fb4afd9a12": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53d082cc3cd845ceb38043e83a259f46": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_940615e80e0e47a987c1286b627553e0",
              "IPY_MODEL_01bc5e17d36f4082966b62b1e4a5a15d",
              "IPY_MODEL_e329322f2f58493695a308cfdee94d36"
            ],
            "layout": "IPY_MODEL_8015d099c2d74f71b1ab4baaf0e52ed5"
          }
        },
        "940615e80e0e47a987c1286b627553e0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5da2e15cb9944ea9b1e15933fd48ea97",
            "placeholder": "​",
            "style": "IPY_MODEL_c859d22b9adf4ac8b50ef5ae372b3114",
            "value": "Downloading (…)neration_config.json: 100%"
          }
        },
        "01bc5e17d36f4082966b62b1e4a5a15d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42f8f7c50f4741cc9a91f9e04feeab6f",
            "max": 124,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_ce04cf8b187f455a97338bb8b48e3e1b",
            "value": 124
          }
        },
        "e329322f2f58493695a308cfdee94d36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1c88fc59c25347729415ed5de5391cc5",
            "placeholder": "​",
            "style": "IPY_MODEL_994e7a97711b4699abf2339ff035b275",
            "value": " 124/124 [00:00&lt;00:00, 4.03kB/s]"
          }
        },
        "8015d099c2d74f71b1ab4baaf0e52ed5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5da2e15cb9944ea9b1e15933fd48ea97": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c859d22b9adf4ac8b50ef5ae372b3114": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "42f8f7c50f4741cc9a91f9e04feeab6f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ce04cf8b187f455a97338bb8b48e3e1b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1c88fc59c25347729415ed5de5391cc5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "994e7a97711b4699abf2339ff035b275": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a41c83de717244768d861d4ea996c0bd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e74856eec41b4e529177eddf7ea31534",
              "IPY_MODEL_0b71e7361a7b40698f9719abc34a0a05",
              "IPY_MODEL_011fbb94acbf4cbf8e512667933cf032"
            ],
            "layout": "IPY_MODEL_9783727f9caf4ed5a6575b188b11e00d"
          }
        },
        "e74856eec41b4e529177eddf7ea31534": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8fda1aac383e4e879bfb941daaa7b18b",
            "placeholder": "​",
            "style": "IPY_MODEL_1d8cc077f31f4407a2d57c483984ffbf",
            "value": "Downloading (…)olve/main/vocab.json: 100%"
          }
        },
        "0b71e7361a7b40698f9719abc34a0a05": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_53e1774694304462a7b1727296f08da7",
            "max": 1042301,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5636b80f9a4c41dbaa8c3d6f5ef9970d",
            "value": 1042301
          }
        },
        "011fbb94acbf4cbf8e512667933cf032": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5dddeb5396054946ba84f9a8d4600b98",
            "placeholder": "​",
            "style": "IPY_MODEL_bde766b72b5c4eb4914c430a03d8e770",
            "value": " 1.04M/1.04M [00:00&lt;00:00, 3.38MB/s]"
          }
        },
        "9783727f9caf4ed5a6575b188b11e00d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8fda1aac383e4e879bfb941daaa7b18b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1d8cc077f31f4407a2d57c483984ffbf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "53e1774694304462a7b1727296f08da7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5636b80f9a4c41dbaa8c3d6f5ef9970d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5dddeb5396054946ba84f9a8d4600b98": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bde766b72b5c4eb4914c430a03d8e770": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f722f88c3d724df982369d246e58700f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_929a357933fb4b82b5f9cc56d2d3ae9e",
              "IPY_MODEL_bcbfd3ebd8be4d12aa308905508636a7",
              "IPY_MODEL_39da12d2a9fd48baaf4254df46edd52c"
            ],
            "layout": "IPY_MODEL_528dd43c4ee042caae4ff69d38064b24"
          }
        },
        "929a357933fb4b82b5f9cc56d2d3ae9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_39cc57a4cd254f3a9a386607843bb161",
            "placeholder": "​",
            "style": "IPY_MODEL_fda6abc530a7493bbefe80f3e43ff728",
            "value": "Downloading (…)olve/main/merges.txt: 100%"
          }
        },
        "bcbfd3ebd8be4d12aa308905508636a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_170b5fca132d4bfca92b680df686bd63",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_66969df88d8843929b38cbc817d3dbd0",
            "value": 456318
          }
        },
        "39da12d2a9fd48baaf4254df46edd52c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_72bb35c36a244e60981b9573a7f40bd1",
            "placeholder": "​",
            "style": "IPY_MODEL_5e1b7abc7e3f4b04bc1dbae3de79d49f",
            "value": " 456k/456k [00:00&lt;00:00, 1.70MB/s]"
          }
        },
        "528dd43c4ee042caae4ff69d38064b24": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "39cc57a4cd254f3a9a386607843bb161": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fda6abc530a7493bbefe80f3e43ff728": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "170b5fca132d4bfca92b680df686bd63": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "66969df88d8843929b38cbc817d3dbd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "72bb35c36a244e60981b9573a7f40bd1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e1b7abc7e3f4b04bc1dbae3de79d49f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9c321528821b46299d5a1e1851916d9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b5260ba3373b48aebe59f03bbd4c226b",
              "IPY_MODEL_2e54448a34e740eda2c2fc7cff80699e",
              "IPY_MODEL_6e0c2d1ed9974611a96a07c1f787d81b"
            ],
            "layout": "IPY_MODEL_954dad7681c743f2b4a6dfc1547dbab7"
          }
        },
        "b5260ba3373b48aebe59f03bbd4c226b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c1b56e76751640e5a2ce9b33a8328da7",
            "placeholder": "​",
            "style": "IPY_MODEL_c0e8f816a6cf434d8aebf58b57378a95",
            "value": "Downloading (…)/main/tokenizer.json: 100%"
          }
        },
        "2e54448a34e740eda2c2fc7cff80699e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_78a550a003114a3c9b44cd1673b35476",
            "max": 1355256,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_593d562071514334af353c071f96b0b5",
            "value": 1355256
          }
        },
        "6e0c2d1ed9974611a96a07c1f787d81b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8eff54fd47864ea1baeba64bf8c7a5c7",
            "placeholder": "​",
            "style": "IPY_MODEL_b0e6727281bf4348aa29a1adf09baf4a",
            "value": " 1.36M/1.36M [00:00&lt;00:00, 4.09MB/s]"
          }
        },
        "954dad7681c743f2b4a6dfc1547dbab7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c1b56e76751640e5a2ce9b33a8328da7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c0e8f816a6cf434d8aebf58b57378a95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "78a550a003114a3c9b44cd1673b35476": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "593d562071514334af353c071f96b0b5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8eff54fd47864ea1baeba64bf8c7a5c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b0e6727281bf4348aa29a1adf09baf4a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/Taaniya/exploring-gpt2-language-model/blob/main/Visualizing_gpt2_token_embeddings.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "This notebook visualizes word token embeddings of GPT2 on Tensorboard projector"
      ],
      "metadata": {
        "id": "g6g-klAfUaqw"
      }
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "96N_eIPxUC6V",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "2f09bb45-8fa9-41fb-f566-3a202dbbdfcf"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.27.4-py3-none-any.whl (6.8 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m6.8/6.8 MB\u001b[0m \u001b[31m44.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.9/dist-packages (from transformers) (3.10.7)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.9/dist-packages (from transformers) (2.27.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.9/dist-packages (from transformers) (4.65.0)\n",
            "Collecting huggingface-hub<1.0,>=0.11.0\n",
            "  Downloading huggingface_hub-0.13.3-py3-none-any.whl (199 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m199.8/199.8 KB\u001b[0m \u001b[31m11.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.9/dist-packages (from transformers) (23.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.9/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (2022.10.31)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.9/dist-packages (from transformers) (1.22.4)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m7.6/7.6 MB\u001b[0m \u001b[31m91.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.9/dist-packages (from huggingface-hub<1.0,>=0.11.0->transformers) (4.5.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (3.4)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (1.26.15)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2.0.12)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.9/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.13.3 tokenizers-0.13.2 transformers-4.27.4\n"
          ]
        }
      ],
      "source": [
        "! pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import GPT2TokenizerFast, GPT2LMHeadModel\n",
        "import tensorflow as tf\n",
        "from tensorboard.plugins import projector\n",
        "\n",
        "import re\n",
        "import os\n",
        "from tqdm import tqdm"
      ],
      "metadata": {
        "id": "U0dnXp76UtFv"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "model = GPT2LMHeadModel.from_pretrained('gpt2')\n",
        "\n",
        "word_embeddings = model.transformer.wte.weight      # Word Token Embeddings \n",
        "position_embeddings = model.transformer.wpe.weight  # Word Position Embeddings "
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "faaf0e5fd82d43c58863f6bdf0f7e838",
            "aae64e9205094a1399d0e6eb19c2b3f9",
            "cde9e650e29a44e2a0a535ebf00e4295",
            "409497a680af41a59e4a322b3fe026b8",
            "a26faaf0e540456a957a72d4a2985adc",
            "e2290dc6664d480686a484da82ee5b36",
            "39b3bd2d816a4c49a03d8f56e6cdabee",
            "f5151b3419fb48a1b638a92d42c9d186",
            "f372db21eefa4e648da1acb00cbcbad0",
            "bb880391a9014cf7ba71d6d4b178b030",
            "06fa0079c51e4f6b85a505944c4734ac",
            "73a0640d1378478180828723624b4f24",
            "aea2750dd1bd43058c503b4b071c5a96",
            "f6e77fcc8854431a981f32f2001155d6",
            "c99b7190a0df4bd195d1cbc7f0d1b5ac",
            "fa60531273324039b35de8f6e1547065",
            "830603676fcc489183b38bcb9b609a50",
            "eed8fc183c6b465789682174ee41c6d1",
            "9b76654088384d99af7e9303fe0ee877",
            "c119b997fc4844f0acadd691089ccee5",
            "3634e4dad04e424faa63d5056bba3f16",
            "8ed1f250917c4e70ba70b2fb4afd9a12",
            "53d082cc3cd845ceb38043e83a259f46",
            "940615e80e0e47a987c1286b627553e0",
            "01bc5e17d36f4082966b62b1e4a5a15d",
            "e329322f2f58493695a308cfdee94d36",
            "8015d099c2d74f71b1ab4baaf0e52ed5",
            "5da2e15cb9944ea9b1e15933fd48ea97",
            "c859d22b9adf4ac8b50ef5ae372b3114",
            "42f8f7c50f4741cc9a91f9e04feeab6f",
            "ce04cf8b187f455a97338bb8b48e3e1b",
            "1c88fc59c25347729415ed5de5391cc5",
            "994e7a97711b4699abf2339ff035b275"
          ]
        },
        "id": "kjHZUGZxUxr9",
        "outputId": "d5496f32-8500-4fbc-aaff-510196ef797d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/665 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "faaf0e5fd82d43c58863f6bdf0f7e838"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading pytorch_model.bin:   0%|          | 0.00/548M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "73a0640d1378478180828723624b4f24"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)neration_config.json:   0%|          | 0.00/124 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "53d082cc3cd845ceb38043e83a259f46"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(word_embeddings.shape)\n",
        "\n",
        "print(position_embeddings.shape)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Cp0C1HiiWEEA",
        "outputId": "7917dc4e-36e9-4295-dcf9-a46f2d2a88d5"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "torch.Size([50257, 768])\n",
            "torch.Size([1024, 768])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# create logging directory\n",
        "log_dir='./logs/vocab/'\n",
        "\n",
        "if not os.path.exists(log_dir):\n",
        "    os.makedirs(log_dir)"
      ],
      "metadata": {
        "id": "7zMc-zK4WJv0"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer = GPT2TokenizerFast.from_pretrained('gpt2')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 113,
          "referenced_widgets": [
            "a41c83de717244768d861d4ea996c0bd",
            "e74856eec41b4e529177eddf7ea31534",
            "0b71e7361a7b40698f9719abc34a0a05",
            "011fbb94acbf4cbf8e512667933cf032",
            "9783727f9caf4ed5a6575b188b11e00d",
            "8fda1aac383e4e879bfb941daaa7b18b",
            "1d8cc077f31f4407a2d57c483984ffbf",
            "53e1774694304462a7b1727296f08da7",
            "5636b80f9a4c41dbaa8c3d6f5ef9970d",
            "5dddeb5396054946ba84f9a8d4600b98",
            "bde766b72b5c4eb4914c430a03d8e770",
            "f722f88c3d724df982369d246e58700f",
            "929a357933fb4b82b5f9cc56d2d3ae9e",
            "bcbfd3ebd8be4d12aa308905508636a7",
            "39da12d2a9fd48baaf4254df46edd52c",
            "528dd43c4ee042caae4ff69d38064b24",
            "39cc57a4cd254f3a9a386607843bb161",
            "fda6abc530a7493bbefe80f3e43ff728",
            "170b5fca132d4bfca92b680df686bd63",
            "66969df88d8843929b38cbc817d3dbd0",
            "72bb35c36a244e60981b9573a7f40bd1",
            "5e1b7abc7e3f4b04bc1dbae3de79d49f",
            "9c321528821b46299d5a1e1851916d9f",
            "b5260ba3373b48aebe59f03bbd4c226b",
            "2e54448a34e740eda2c2fc7cff80699e",
            "6e0c2d1ed9974611a96a07c1f787d81b",
            "954dad7681c743f2b4a6dfc1547dbab7",
            "c1b56e76751640e5a2ce9b33a8328da7",
            "c0e8f816a6cf434d8aebf58b57378a95",
            "78a550a003114a3c9b44cd1673b35476",
            "593d562071514334af353c071f96b0b5",
            "8eff54fd47864ea1baeba64bf8c7a5c7",
            "b0e6727281bf4348aa29a1adf09baf4a"
          ]
        },
        "id": "ArljXsF6Xghy",
        "outputId": "d01b75b7-7c67-408c-c799-4c97d0e9393b"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/vocab.json:   0%|          | 0.00/1.04M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a41c83de717244768d861d4ea996c0bd"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)olve/main/merges.txt:   0%|          | 0.00/456k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f722f88c3d724df982369d246e58700f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)/main/tokenizer.json:   0%|          | 0.00/1.36M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9c321528821b46299d5a1e1851916d9f"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.pretrained_vocab_files_map"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "0BNsjcZJXmYd",
        "outputId": "6a3ea0ab-cd2a-46ea-cd4c-72924588b06f"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'vocab_file': {'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',\n",
              "  'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',\n",
              "  'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',\n",
              "  'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',\n",
              "  'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json'},\n",
              " 'merges_file': {'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',\n",
              "  'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',\n",
              "  'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',\n",
              "  'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',\n",
              "  'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt'},\n",
              " 'tokenizer_file': {'gpt2': 'https://huggingface.co/gpt2/resolve/main/tokenizer.json',\n",
              "  'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json',\n",
              "  'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/tokenizer.json',\n",
              "  'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json',\n",
              "  'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/tokenizer.json'}}"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "tokenizer.vocab"
      ],
      "metadata": {
        "id": "xoxI5CLLX962",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a8a5f769-4322-4573-d6d0-8ab445a5b08d"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'German': 16010,\n",
              " 'ICS': 19505,\n",
              " 'ĠPotter': 14179,\n",
              " 'Ġprosecutor': 13683,\n",
              " 'Ġmanslaughter': 35083,\n",
              " 'Ġget': 651,\n",
              " '701': 41583,\n",
              " 'Ġinserts': 42220,\n",
              " 'Phys': 43215,\n",
              " 'udd': 4185,\n",
              " 'Ġchildhood': 9963,\n",
              " 'ĠBaker': 14372,\n",
              " 'Ġacademy': 21531,\n",
              " 'Ġrelax': 8960,\n",
              " 'Ġbooking': 25452,\n",
              " 'Ġoppressed': 25765,\n",
              " '644': 29173,\n",
              " 'ilar': 1794,\n",
              " 'Ġwake': 7765,\n",
              " 'ized': 1143,\n",
              " 'ĠBarn': 11842,\n",
              " 'Ġallowing': 5086,\n",
              " 'ĠâĶĤ': 19421,\n",
              " 'ĠSch': 3059,\n",
              " 'orthodox': 42539,\n",
              " 'ĠSergeant': 26541,\n",
              " 'Supplement': 42615,\n",
              " '993': 44821,\n",
              " 'Ġhugging': 46292,\n",
              " 'åį': 39355,\n",
              " 'Ġlegendary': 13273,\n",
              " 'Ġprize': 11596,\n",
              " 'FB': 26001,\n",
              " 'DP': 6322,\n",
              " 'Ġdrummer': 34269,\n",
              " 'Ġhistorically': 15074,\n",
              " 'ĠPhysical': 16331,\n",
              " 'Ġdeterior': 17975,\n",
              " 'ĠRoot': 20410,\n",
              " 'Ġparasite': 38473,\n",
              " 'uci': 42008,\n",
              " 'ĠTransgender': 44002,\n",
              " '244': 25707,\n",
              " 'Ġbaskets': 46530,\n",
              " 'Ġcost': 1575,\n",
              " 'Ġback': 736,\n",
              " 'Ġ164': 25307,\n",
              " 'Ġnow': 783,\n",
              " 'Ġpie': 2508,\n",
              " 'Ġstarted': 2067,\n",
              " 'ĠBard': 25654,\n",
              " 'aez': 47246,\n",
              " '774': 47582,\n",
              " 'Ġconfid': 47830,\n",
              " 'ĠSeeking': 48160,\n",
              " 'Ġemploys': 24803,\n",
              " 'ĠWhich': 9022,\n",
              " 'ĠWeb': 5313,\n",
              " 'ĠMash': 30870,\n",
              " 'Ġsuccess': 1943,\n",
              " 'Ġcommits': 23463,\n",
              " 'susp': 40409,\n",
              " 'ĠLAR': 47211,\n",
              " 'yk': 48361,\n",
              " 'CRIP': 36584,\n",
              " 'Ġameric': 45630,\n",
              " 'Device': 24728,\n",
              " 'Ġaust': 38132,\n",
              " 'Ġfactual': 27234,\n",
              " 'Ty': 25492,\n",
              " 'Pers': 30946,\n",
              " 'Ġnegatives': 42510,\n",
              " 'ness': 1108,\n",
              " 'Ġsto': 3995,\n",
              " 'ooth': 5226,\n",
              " 'Malley': 33776,\n",
              " 'Evidence': 46785,\n",
              " 'ĠRussell': 11563,\n",
              " 'ĠDrake': 18515,\n",
              " 'Ġratios': 22423,\n",
              " 'Ġ1914': 26833,\n",
              " 'ĠMarlins': 38134,\n",
              " 'ĠVariety': 39143,\n",
              " 'Ġsupervision': 20865,\n",
              " 'ĠCov': 39751,\n",
              " 'Ġveterinary': 38435,\n",
              " 'Ġglow': 19634,\n",
              " 'ĠRevolutionary': 28105,\n",
              " 'ĠHardware': 28715,\n",
              " 'ĠVirginia': 6025,\n",
              " 'Ġowned': 6898,\n",
              " 'measures': 47336,\n",
              " 'mare': 11449,\n",
              " 'ĠToyota': 20182,\n",
              " 'ĠBaxter': 46277,\n",
              " 'Ġstick': 4859,\n",
              " 'Â©': 16224,\n",
              " 'Foot': 17574,\n",
              " 'Ġapprovals': 45818,\n",
              " 'Transform': 41762,\n",
              " 'Ġthese': 777,\n",
              " 'Purchase': 47651,\n",
              " 'ĠElder': 15624,\n",
              " 'Ġprone': 17592,\n",
              " 'Ġcloth': 16270,\n",
              " 'ĠSaga': 19743,\n",
              " 'ĠNFC': 24004,\n",
              " '1982': 30763,\n",
              " 'Ġobsess': 24961,\n",
              " 'ĠFernando': 31063,\n",
              " 'girls': 36960,\n",
              " 'particip': 48013,\n",
              " 'tesy': 27090,\n",
              " 'Ġancestor': 31836,\n",
              " 'fw': 44482,\n",
              " 'hello': 31373,\n",
              " 'ONE': 11651,\n",
              " 'ĠFors': 27325,\n",
              " 'ĠFrank': 5278,\n",
              " 'Ġinjured': 6686,\n",
              " 'uer': 15573,\n",
              " 'lict': 13758,\n",
              " 'ĠYang': 10998,\n",
              " 'Ġdoses': 16376,\n",
              " 'Ġincident': 4519,\n",
              " 'cz': 26691,\n",
              " 'Ġacquire': 12831,\n",
              " 'ĠParkway': 38613,\n",
              " 'Cost': 13729,\n",
              " 'Ġtransgress': 44780,\n",
              " 'ĠRadio': 8829,\n",
              " 'Ķ': 242,\n",
              " 'Gil': 40747,\n",
              " 'ija': 34655,\n",
              " 'Ġinvade': 30419,\n",
              " 'Ġspiked': 38927,\n",
              " 'Ġalerts': 21675,\n",
              " 'Ġreliever': 48460,\n",
              " 'Rated': 15322,\n",
              " 'Ġimpatient': 33440,\n",
              " 'ibilities': 7992,\n",
              " 'Ġace': 31506,\n",
              " 'Ġoperational': 13919,\n",
              " '&': 5,\n",
              " 'Ġlore': 24044,\n",
              " 'is': 271,\n",
              " 'key': 2539,\n",
              " 'hon': 24130,\n",
              " 'ĠTray': 34575,\n",
              " 'Ġhanged': 40597,\n",
              " 'Ġmedical': 3315,\n",
              " 'ĠBottle': 33608,\n",
              " 'Ġcontrovers': 5467,\n",
              " '886': 44980,\n",
              " 'Ġgenerating': 15453,\n",
              " 'ĠLeaves': 46597,\n",
              " 'painted': 47351,\n",
              " 'ĠSeb': 20639,\n",
              " 'ĠThen': 3244,\n",
              " 'Ġcoat': 13209,\n",
              " 'Ð»': 30143,\n",
              " 'Ġuncertain': 8627,\n",
              " 'ANI': 43664,\n",
              " 'Match': 23850,\n",
              " 'Ġinscribed': 45100,\n",
              " 'Ġmath': 10688,\n",
              " 'ountain': 18635,\n",
              " 'Ġdepressive': 36568,\n",
              " 'ĠDoomsday': 45234,\n",
              " 'Ġflushed': 44869,\n",
              " 'Ġpreserve': 12201,\n",
              " 'rap': 2416,\n",
              " 'otos': 14163,\n",
              " 'ĠAuburn': 24579,\n",
              " 'Ġsexism': 24953,\n",
              " 'Ġobscure': 18611,\n",
              " 'Ġunion': 6441,\n",
              " 'Ġbridges': 19432,\n",
              " '217': 24591,\n",
              " 'Ġlatch': 43140,\n",
              " 'mc': 23209,\n",
              " 'ĠScot': 7130,\n",
              " 'Ġpainted': 13055,\n",
              " 'LGBT': 37701,\n",
              " 'ĠNathaniel': 49536,\n",
              " 'EST': 6465,\n",
              " 'iolet': 19194,\n",
              " 'Ġadventurer': 40994,\n",
              " 'Wave': 39709,\n",
              " 'Ġfermented': 44543,\n",
              " 'Ġorganising': 46729,\n",
              " 'Ġgle': 26852,\n",
              " 'ĠVendor': 39896,\n",
              " 'ble': 903,\n",
              " 'igmat': 32441,\n",
              " 'eed': 2308,\n",
              " 'ĠReal': 6416,\n",
              " 'Ġdrill': 16007,\n",
              " 'Ġcit': 15433,\n",
              " 'Ġresolutions': 21811,\n",
              " 'Ġ159': 26422,\n",
              " 'ĠXV': 29922,\n",
              " 'Ġhp': 27673,\n",
              " 'cker': 15280,\n",
              " 'Ġincarceration': 27224,\n",
              " 'Ġbids': 27837,\n",
              " 'Ġcriticism': 7734,\n",
              " 'Ġfurther': 2252,\n",
              " 'Ġchip': 11594,\n",
              " 'Ġearnest': 23176,\n",
              " 'Ġax': 7877,\n",
              " 'unks': 14125,\n",
              " 'ĠSiber': 28394,\n",
              " 'ĠCaroline': 29037,\n",
              " 'ĠDocuments': 33267,\n",
              " 'Ġdelicate': 19217,\n",
              " 'Ġdisciplines': 29861,\n",
              " 'Ġcheers': 34550,\n",
              " 'Ġover': 625,\n",
              " 'riad': 21244,\n",
              " 'Ġnat': 34664,\n",
              " 'Ġacquainted': 36620,\n",
              " 'Ġarbitration': 26038,\n",
              " 'ĠÃ©': 38251,\n",
              " 'rontal': 39321,\n",
              " 'Ġpilgr': 23177,\n",
              " 'Ġmeditation': 16901,\n",
              " 'Ġhosp': 10496,\n",
              " 'translation': 41519,\n",
              " 'ĠPearce': 41902,\n",
              " 'iant': 3014,\n",
              " 'ipop': 42800,\n",
              " 'Ġpoints': 2173,\n",
              " 'Ġuntil': 1566,\n",
              " 'flake': 47597,\n",
              " '288': 25270,\n",
              " 'Ġcurfew': 49565,\n",
              " 'JSON': 40386,\n",
              " 'amac': 11494,\n",
              " 'Ġanalyses': 13523,\n",
              " 'ĠBronze': 19461,\n",
              " 'Â¨': 37102,\n",
              " 'iatures': 42711,\n",
              " 'ĠWorcester': 50075,\n",
              " 'illance': 7682,\n",
              " 'aters': 8605,\n",
              " 'ĠMoe': 46488,\n",
              " 'APTER': 29485,\n",
              " 'UK': 15039,\n",
              " 'âĸĪâĸĪ': 9968,\n",
              " 'ript': 1968,\n",
              " 'ĠKee': 19799,\n",
              " 'Ġspiritual': 8557,\n",
              " 'ential': 1843,\n",
              " 'Ġeasily': 3538,\n",
              " 'Ġbranches': 13737,\n",
              " 'Ġstarring': 20495,\n",
              " 'Ġtemporal': 21964,\n",
              " 'Ġtanks': 11657,\n",
              " 'acey': 25415,\n",
              " 'inctions': 31253,\n",
              " 'adia': 29523,\n",
              " 'ĠShuttle': 35143,\n",
              " 'Ġweapon': 4282,\n",
              " 'andem': 30025,\n",
              " 'Ġpits': 29699,\n",
              " 'produ': 18230,\n",
              " 'Program': 15167,\n",
              " 'Ġleng': 40038,\n",
              " 'ulating': 8306,\n",
              " 'Ġnexus': 45770,\n",
              " 'Ġhoax': 25943,\n",
              " 'ener': 877,\n",
              " 'phone': 4862,\n",
              " 'ĠMoto': 23049,\n",
              " '366': 32459,\n",
              " 'acles': 9928,\n",
              " 'India': 21569,\n",
              " 'ĠMillennials': 42828,\n",
              " 'Simon': 35475,\n",
              " 'Ġhone': 47267,\n",
              " 'Ġlisten': 6004,\n",
              " 'Ġsore': 19597,\n",
              " 'conference': 41124,\n",
              " 'Ġbox': 3091,\n",
              " 'Ġembedded': 14553,\n",
              " 'ĠDN': 45080,\n",
              " 'Í': 137,\n",
              " 'tex': 16886,\n",
              " 'Ġsucc': 17458,\n",
              " 'tele': 46813,\n",
              " 'ĠParm': 47796,\n",
              " 'Ġroot': 6808,\n",
              " 'ĠMechanical': 19663,\n",
              " 'callback': 47423,\n",
              " 'Ġgrab': 5552,\n",
              " 'col': 4033,\n",
              " 'Ġextent': 6287,\n",
              " 'RY': 18276,\n",
              " 'Ġchests': 34572,\n",
              " 'Ġshrugged': 28271,\n",
              " 'Ġcommun': 1316,\n",
              " 'drops': 49253,\n",
              " 'Ġbinds': 37354,\n",
              " 'Few': 32351,\n",
              " 'ĠMG': 34809,\n",
              " 'ĠDarth': 31152,\n",
              " 'melon': 45690,\n",
              " 'ĠGow': 41396,\n",
              " 'Ġfake': 8390,\n",
              " 'ĠSharif': 44144,\n",
              " 'Ġremembrance': 44827,\n",
              " 'Ġ280': 21355,\n",
              " 'quet': 21108,\n",
              " 'Ġsuspect': 4099,\n",
              " 'ĠParadox': 37983,\n",
              " 'Ġreun': 15398,\n",
              " 'cano': 35490,\n",
              " 'pless': 14570,\n",
              " 'Ġworks': 2499,\n",
              " 'Comp': 7293,\n",
              " 'maker': 10297,\n",
              " 'Ġwaved': 26834,\n",
              " 'Ġhast': 19338,\n",
              " 'ĠSen': 2311,\n",
              " 'ĠCultural': 23897,\n",
              " 'Ġfreshmen': 45208,\n",
              " 'Ġ[+]': 47175,\n",
              " 'Ġwebcam': 49823,\n",
              " 'Ġexpl': 1193,\n",
              " 'Ġcaptures': 23007,\n",
              " 'ĠSop': 35643,\n",
              " 'xml': 19875,\n",
              " 'ĠFormation': 46689,\n",
              " 'Ġgrounds': 9384,\n",
              " 'Ġexploitation': 17238,\n",
              " 'gang': 28284,\n",
              " 'Ġinforming': 29140,\n",
              " 'ĠAntonio': 11366,\n",
              " 'Taking': 26556,\n",
              " '040': 36676,\n",
              " 'ĠDollars': 45792,\n",
              " 'Ġastronomical': 37209,\n",
              " '===': 18604,\n",
              " 'Ġmost': 749,\n",
              " 'Ġmounted': 12623,\n",
              " 'ĠRandom': 14534,\n",
              " 'Ġbroom': 47085,\n",
              " 'Ġinterpre': 16795,\n",
              " 'Ġaver': 9076,\n",
              " 'casts': 40924,\n",
              " 'Ġpassive': 14513,\n",
              " 'Ġsnaps': 23429,\n",
              " 'Ġunfairly': 30750,\n",
              " 'Ġinept': 43891,\n",
              " 'Ġlonely': 21757,\n",
              " 'Ġlease': 15278,\n",
              " 'Ġidentifiable': 34450,\n",
              " 'Ġprematurely': 41370,\n",
              " 'Î³': 42063,\n",
              " 'Ġinherit': 16955,\n",
              " 'Ġloss': 2994,\n",
              " 'Ġdemonstration': 13646,\n",
              " 'Ġrou': 13805,\n",
              " 'IVE': 9306,\n",
              " 'ĠRepresentatives': 17132,\n",
              " '/(': 29006,\n",
              " 'appings': 39242,\n",
              " 'Ġunveiling': 41438,\n",
              " 'Ġspectral': 37410,\n",
              " 'Ġwhether': 1771,\n",
              " 'Ġsquares': 24438,\n",
              " 'Ġstrokes': 29483,\n",
              " 'Ġcalc': 42302,\n",
              " 'ĠSa': 10318,\n",
              " 'Ġredund': 22916,\n",
              " 'Ġlonger': 2392,\n",
              " 'Ġitiner': 45142,\n",
              " 'Ġdeer': 20096,\n",
              " 'ĠInter': 4225,\n",
              " 'ĠLeo': 19632,\n",
              " '600': 8054,\n",
              " 'Ġjung': 34799,\n",
              " 'Mars': 43725,\n",
              " 'Kelly': 34831,\n",
              " 'ĠAnalog': 50088,\n",
              " 'Ġsays': 1139,\n",
              " 'ĠHang': 24300,\n",
              " 'iovascular': 19381,\n",
              " 'Ġ260': 21148,\n",
              " 'ĠAccess': 8798,\n",
              " 'neys': 20141,\n",
              " 'ĠHyp': 21209,\n",
              " 'Ġjohn': 45610,\n",
              " 'Ġnights': 12513,\n",
              " 'Ret': 9781,\n",
              " 'arettes': 13890,\n",
              " 'Ġeasier': 4577,\n",
              " 'ĠRoth': 16131,\n",
              " 'Ġincarcerated': 33946,\n",
              " 'Ġlargely': 5688,\n",
              " 'Ġproceedings': 13106,\n",
              " 'ĠJournals': 48608,\n",
              " 'bodied': 45190,\n",
              " 'emis': 30561,\n",
              " 'imgur': 19791,\n",
              " 'Ġvodka': 36279,\n",
              " 'Ġselects': 40573,\n",
              " 'Ġresided': 49330,\n",
              " 'pin': 11635,\n",
              " 'ĠVince': 22376,\n",
              " 'Ġturbo': 29292,\n",
              " 'ĠLah': 30693,\n",
              " 'ĠDor': 12528,\n",
              " 'iphany': 49915,\n",
              " 'Move': 21774,\n",
              " 'patch': 17147,\n",
              " 'Jere': 31579,\n",
              " 'ĠSomebody': 43141,\n",
              " 'hler': 49737,\n",
              " 'involved': 44697,\n",
              " 'Ġma': 17266,\n",
              " 'raphic': 22262,\n",
              " 'lists': 20713,\n",
              " 'Ġimpact': 2928,\n",
              " 'ĠOlympics': 14935,\n",
              " 'Ġchess': 19780,\n",
              " 'ĠSz': 27974,\n",
              " 'ĠLM': 37125,\n",
              " 'Ġstagnant': 41391,\n",
              " 'auldron': 45637,\n",
              " 'umb': 2178,\n",
              " 'Ġadop': 4344,\n",
              " 'Ġbehalf': 8378,\n",
              " 'Ġdual': 10668,\n",
              " 'ĠShip': 16656,\n",
              " 'Mouse': 39643,\n",
              " 'Ġaccept': 2453,\n",
              " 'ĠStaten': 46034,\n",
              " 'orrow': 6254,\n",
              " 'Ġsuspending': 47499,\n",
              " 'y': 88,\n",
              " 'ĠUntil': 14303,\n",
              " 'Ġslipping': 29612,\n",
              " 'ĠPhil': 4543,\n",
              " 'Ġè': 5525,\n",
              " 'ĠguiActive': 18472,\n",
              " 'Ġborrower': 39740,\n",
              " 'Ġeditions': 23713,\n",
              " 'Ġ1865': 47801,\n",
              " 'Ġclimate': 4258,\n",
              " 'ITT': 22470,\n",
              " 'ĠNYC': 19170,\n",
              " 'Items': 23022,\n",
              " 'ÑĮ': 45367,\n",
              " 'ĠMaze': 33412,\n",
              " 'udes': 8401,\n",
              " 'ĠSHARES': 44806,\n",
              " 'Ġswapping': 38869,\n",
              " 'ĠDollar': 29747,\n",
              " 'Open': 11505,\n",
              " 'paid': 20333,\n",
              " 'ĠPull': 21429,\n",
              " 'Ġaph': 49812,\n",
              " 'dinand': 41993,\n",
              " 'Ġclosures': 32149,\n",
              " 'ĠYour': 3406,\n",
              " 'front': 8534,\n",
              " 'ĠProtection': 9985,\n",
              " 'forth': 25718,\n",
              " 'Ġseniors': 23481,\n",
              " 'Ġpetition': 8853,\n",
              " 'tainment': 10738,\n",
              " 'Ġmaintains': 16047,\n",
              " 'Ã§': 16175,\n",
              " 'ophical': 49256,\n",
              " 'Ġ160': 13454,\n",
              " 'ĠHills': 14379,\n",
              " 'Ġunre': 14880,\n",
              " 'Ġstood': 6204,\n",
              " 'Ġtops': 21246,\n",
              " 'arching': 38270,\n",
              " 'adden': 38014,\n",
              " 'Ġtoured': 44035,\n",
              " 'ĠVoice': 15282,\n",
              " 'ĠCertificate': 27895,\n",
              " 'snap': 45380,\n",
              " 'Ġcounterpart': 11283,\n",
              " 'ĠResearchers': 26685,\n",
              " 'Ġremed': 20100,\n",
              " 'Ġcombust': 27500,\n",
              " 'ĠHalloween': 16032,\n",
              " 'Ġpartly': 11476,\n",
              " 'Ġmandates': 33568,\n",
              " 'Ġpants': 12581,\n",
              " 'Bind': 36180,\n",
              " 'ĠMarcel': 36547,\n",
              " 'Ġsocio': 21790,\n",
              " 'algia': 47111,\n",
              " 'Ġcryptic': 47774,\n",
              " 'Ġgelatin': 49084,\n",
              " 'Harry': 18308,\n",
              " 'safe': 21230,\n",
              " 'ĠThompson': 11654,\n",
              " 'mes': 6880,\n",
              " 'ĠKurd': 9778,\n",
              " 'loaded': 14578,\n",
              " 'Ă': 190,\n",
              " 'Ġrulings': 31532,\n",
              " 'Ġserotonin': 32236,\n",
              " 'ĠMinotaur': 40558,\n",
              " 'Ġkick': 4829,\n",
              " 'agher': 30450,\n",
              " 'sq': 31166,\n",
              " 'ĠTruman': 33027,\n",
              " 'Â»': 17730,\n",
              " 'ĠKoen': 49178,\n",
              " 'ĠMobil': 45951,\n",
              " 'ĠMarsh': 9786,\n",
              " 'max': 9806,\n",
              " 'Ġsacrificed': 27445,\n",
              " 'ĠAngle': 42375,\n",
              " 'udi': 47928,\n",
              " 'iked': 17951,\n",
              " 'Ġdecre': 5255,\n",
              " 'Ġdistractions': 38085,\n",
              " 'ed': 276,\n",
              " 'Ġmagical': 10883,\n",
              " 'Ġjab': 33896,\n",
              " 'Ġmustard': 33694,\n",
              " 'ĠBlossom': 35544,\n",
              " 'ĠãĤµ': 38390,\n",
              " 'Ġwarn': 9828,\n",
              " 'hig': 25196,\n",
              " 'gently': 34727,\n",
              " 'Ġtruce': 39854,\n",
              " 'Motor': 34919,\n",
              " 'achable': 34446,\n",
              " 'driver': 26230,\n",
              " 'Camera': 35632,\n",
              " 'Ġpolarization': 42704,\n",
              " 'Ġnp': 45941,\n",
              " 'ĠGon': 35371,\n",
              " 'Ġcontrolling': 12755,\n",
              " 'Ġunsigned': 22165,\n",
              " 'ĠGauntlet': 48941,\n",
              " 'ĠValid': 48951,\n",
              " 'ĠCOVER': 47902,\n",
              " 'prototype': 38124,\n",
              " 'risis': 42841,\n",
              " '42': 3682,\n",
              " '18': 1507,\n",
              " 'Ġdisguise': 27487,\n",
              " 'Ġparish': 28830,\n",
              " 'Ġisland': 7022,\n",
              " 'ĠList': 7343,\n",
              " 'Ġreactive': 32242,\n",
              " 'zai': 35142,\n",
              " 'ĠJeep': 35801,\n",
              " 'peror': 8723,\n",
              " '784': 37688,\n",
              " 'Ġko': 41727,\n",
              " 'ĠInspect': 20904,\n",
              " 'Ġproducts': 3186,\n",
              " 'Ġnails': 23361,\n",
              " 'hammad': 14875,\n",
              " 'ĠIris': 34230,\n",
              " 'Tro': 44095,\n",
              " 'Ġpious': 45979,\n",
              " 'ĠOutput': 25235,\n",
              " 'ĠMold': 29684,\n",
              " 'Ġgardens': 25476,\n",
              " 'Ġequity': 13598,\n",
              " 'ĠDefinitely': 33562,\n",
              " 'Ġbigotry': 28849,\n",
              " 'Ġformulation': 31760,\n",
              " 'generic': 41357,\n",
              " 'ĠSER': 18871,\n",
              " 'Ġdarkness': 11854,\n",
              " 'Ġ122': 19409,\n",
              " 'places': 23625,\n",
              " 'ĠCyr': 40399,\n",
              " 'attering': 16475,\n",
              " 'ĠScalia': 32575,\n",
              " 'Ġdecorative': 39798,\n",
              " 'ĠVeteran': 32733,\n",
              " 'Ġneutral': 8500,\n",
              " 'ĠPlatinum': 23851,\n",
              " 'Such': 16678,\n",
              " 'Ġhonor': 7522,\n",
              " 'Ġsuspic': 12486,\n",
              " 'Ġadvances': 14901,\n",
              " 'uary': 2838,\n",
              " 'assing': 19696,\n",
              " 'ational': 864,\n",
              " 'Ġtide': 20013,\n",
              " 'Ġattempt': 2230,\n",
              " 'fore': 754,\n",
              " 'Ġjuicy': 35075,\n",
              " 'ĠMysterious': 40981,\n",
              " 'ĠNAV': 47140,\n",
              " 'Ġamplify': 47366,\n",
              " 'ophen': 47806,\n",
              " 'olan': 16617,\n",
              " 'Ġdeposits': 18190,\n",
              " 'ultz': 22150,\n",
              " '1964': 46477,\n",
              " 'Ġghetto': 46482,\n",
              " 'Gb': 49017,\n",
              " 'Ġtraveller': 49130,\n",
              " 'Ġsatisfied': 11378,\n",
              " 'Ġplayable': 22469,\n",
              " 'Ġ191': 31009,\n",
              " 'Ġesp': 15024,\n",
              " 'Ġindividually': 17033,\n",
              " 'Termin': 44798,\n",
              " 'emetery': 19785,\n",
              " 'Vi': 38432,\n",
              " 'ĠBank': 5018,\n",
              " 'Ġcapacitor': 43212,\n",
              " 'autions': 28766,\n",
              " 'Cas': 35155,\n",
              " 'db': 9945,\n",
              " 'Ġding': 44852,\n",
              " 'OLD': 15173,\n",
              " '267': 25674,\n",
              " 'hw': 36599,\n",
              " 'ĠSaturn': 23135,\n",
              " 'Ġsteel': 7771,\n",
              " 'ĠTr': 833,\n",
              " 'Ġvill': 4489,\n",
              " 'Ġdiligently': 44243,\n",
              " 'ĠDog': 8532,\n",
              " 'Ġfumes': 47301,\n",
              " 'Ġ::': 7904,\n",
              " 'Ġ315': 32647,\n",
              " 'ib': 571,\n",
              " 'Ġbuilders': 31606,\n",
              " 'Ġshot': 2823,\n",
              " 'acers': 49908,\n",
              " 'Ġmotorists': 32688,\n",
              " 'ĠPul': 21624,\n",
              " '%.': 7225,\n",
              " 'Update': 10260,\n",
              " 'Ġvitamin': 14411,\n",
              " 'Ġdivorced': 25107,\n",
              " 'Ġgrabbed': 13176,\n",
              " 'Ġbuddies': 35548,\n",
              " 'ĠFlower': 20025,\n",
              " 'Ġrestoring': 25646,\n",
              " 'Ġtimber': 30605,\n",
              " 'meaning': 24815,\n",
              " 'ĠKaz': 16385,\n",
              " 'Ġseeming': 31792,\n",
              " 'Ġarc': 10389,\n",
              " 'Ġeliminates': 32311,\n",
              " 'ĠRomance': 36555,\n",
              " 'ĠDeck': 20961,\n",
              " 'ĠCa': 6488,\n",
              " 'Ġaliens': 16269,\n",
              " 'Ġ171': 28369,\n",
              " 'Ġ183': 28551,\n",
              " 'solete': 23869,\n",
              " 'Position': 26545,\n",
              " 'ĠCoyotes': 42228,\n",
              " 'Ġunreliable': 29954,\n",
              " 'Ġrights': 2489,\n",
              " 'Conf': 18546,\n",
              " 'Ġally': 12525,\n",
              " 'Ġpuppies': 37793,\n",
              " 'Ġresultant': 43440,\n",
              " 'ĠUtt': 36057,\n",
              " 'scoring': 46536,\n",
              " 'SQL': 17861,\n",
              " 'ĠGTX': 12646,\n",
              " 'ĠCycle': 26993,\n",
              " 'vidia': 21744,\n",
              " 'Ġmaintaining': 10941,\n",
              " 'Ġpleaded': 16407,\n",
              " 'ĠFellowship': 36293,\n",
              " 'ĠRory': 37178,\n",
              " 'Ġdisag': 7969,\n",
              " 'ath': 776,\n",
              " 'PORT': 15490,\n",
              " '280': 21033,\n",
              " 'ĠPod': 17437,\n",
              " 'arton': 41328,\n",
              " 'Ġreadings': 24654,\n",
              " 'Ġillegitimate': 41368,\n",
              " 'WARE': 33746,\n",
              " 'videos': 32861,\n",
              " 'Ġledge': 35614,\n",
              " 'Ġ./': 24457,\n",
              " 'CIA': 49732,\n",
              " 'ĠXbox': 9445,\n",
              " 'chemical': 31379,\n",
              " '99': 2079,\n",
              " 'ierrez': 44448,\n",
              " 'ĠDeaths': 37044,\n",
              " 'FREE': 39274,\n",
              " 'Ġshowed': 3751,\n",
              " 'Ġwarmth': 23125,\n",
              " 'ĠKob': 23403,\n",
              " 'Ġvanity': 39609,\n",
              " 'Syn': 29934,\n",
              " 'Ġneutrality': 20723,\n",
              " 'Ġpools': 20354,\n",
              " 'ĠStud': 3604,\n",
              " 'ĠVersion': 10628,\n",
              " '789': 40401,\n",
              " 'ĠBots': 40946,\n",
              " 'Ġticks': 36066,\n",
              " 'Ġegregious': 34372,\n",
              " 'Ġwoodland': 49609,\n",
              " 'apsed': 28361,\n",
              " 'Ġproven': 9157,\n",
              " 'itimate': 30233,\n",
              " 'ĠPret': 37123,\n",
              " 'ĠSkin': 17847,\n",
              " 'ĠGin': 21444,\n",
              " 'Ġgenerations': 10439,\n",
              " 'ĠWade': 19136,\n",
              " 'pleted': 16838,\n",
              " 'jack': 19650,\n",
              " 'Ġunfolding': 34502,\n",
              " 'alli': 36546,\n",
              " '\"...': 26214,\n",
              " 'ĠJiu': 42310,\n",
              " 'host': 4774,\n",
              " 'ĠGlac': 42719,\n",
              " 'ucker': 12603,\n",
              " 'Ġovercrowd': 38080,\n",
              " 'ĠAadhaar': 47727,\n",
              " 'ĠSilk': 26006,\n",
              " 'Ġgenesis': 48861,\n",
              " 'Ġreduce': 4646,\n",
              " 'Ġdrank': 24070,\n",
              " 'ĠEnvironment': 9344,\n",
              " 'ĠIvy': 27386,\n",
              " 'Ġforegoing': 41660,\n",
              " 'Ġsilent': 10574,\n",
              " 'ĠPerkins': 29618,\n",
              " 'PP': 10246,\n",
              " 'ĠAer': 15781,\n",
              " 'Ġ$_': 40111,\n",
              " 'Ġ358': 41761,\n",
              " '031': 43637,\n",
              " 'atted': 16898,\n",
              " 'Ġphysiology': 38033,\n",
              " 'Ġgovernors': 26824,\n",
              " 'Ġcorridor': 20749,\n",
              " 'Ġtou': 2819,\n",
              " 'endas': 35624,\n",
              " 'Way': 25309,\n",
              " 'Ġcha': 17792,\n",
              " 'ãĥĲ': 29659,\n",
              " 'Ġdisl': 19621,\n",
              " 'atic': 1512,\n",
              " 'ĠDill': 44322,\n",
              " 'Ġrestrictions': 8733,\n",
              " 'BO': 8202,\n",
              " 'Ġeconomy': 3773,\n",
              " 'atur': 2541,\n",
              " 'above': 29370,\n",
              " 'Ġ436': 50038,\n",
              " 'Ġwore': 12408,\n",
              " 'Ġwhence': 44012,\n",
              " 'Ġ999': 36006,\n",
              " 'Ġunmist': 40016,\n",
              " 'illus': 44342,\n",
              " 'Fla': 47487,\n",
              " 'ĠM': 337,\n",
              " 'Ġnucle': 17751,\n",
              " 'ĠGreenland': 30155,\n",
              " 'sector': 34914,\n",
              " 'Ġ4096': 42479,\n",
              " 'Ġyours': 12431,\n",
              " 'mates': 7300,\n",
              " 'ournaments': 16950,\n",
              " 'chen': 6607,\n",
              " 'Ġtraffic': 4979,\n",
              " 'Ġfascinating': 13899,\n",
              " 'Ġmissile': 10105,\n",
              " 'Ġshar': 13986,\n",
              " 'Ġshocking': 14702,\n",
              " 'Ġrelease': 2650,\n",
              " 'Ġweights': 19590,\n",
              " 'Ġshrinking': 27382,\n",
              " 'Ġbits': 10340,\n",
              " 'quire': 29782,\n",
              " '019': 30484,\n",
              " 'bj': 50007,\n",
              " 'ĠAncient': 13406,\n",
              " 'Ġcartridge': 26904,\n",
              " 'resa': 14625,\n",
              " 'ĠArticles': 22698,\n",
              " 'Me': 5308,\n",
              " 'Ġestate': 7964,\n",
              " 'aman': 10546,\n",
              " 'elta': 12514,\n",
              " 'ĠBrowns': 17671,\n",
              " '================================================================': 23926,\n",
              " 'ĠGoff': 44572,\n",
              " 'ocusing': 45743,\n",
              " 'Ġpopup': 46207,\n",
              " 'idan': 27610,\n",
              " 'outube': 9762,\n",
              " 'ĠIzan': 35144,\n",
              " 'ĠSTOP': 44934,\n",
              " 'Ġsol': 1540,\n",
              " 'ĠRodgers': 19947,\n",
              " 'Ġhonoured': 45870,\n",
              " 'ĠPitt': 10276,\n",
              " '581': 48630,\n",
              " 'licks': 49191,\n",
              " 'Ġanalys': 11090,\n",
              " 'ĠScore': 15178,\n",
              " 'Dr': 6187,\n",
              " 'Ġsaving': 8914,\n",
              " 'Whe': 10842,\n",
              " 'ĠBloom': 11891,\n",
              " 'Player': 14140,\n",
              " 'ĠCash': 16210,\n",
              " 'Ġshock': 6380,\n",
              " 'Ġjewelry': 22634,\n",
              " 'ĠMent': 31879,\n",
              " 'LinkedIn': 40574,\n",
              " 'che': 2395,\n",
              " 'éĹ': 29785,\n",
              " 'REM': 40726,\n",
              " 'Ġhandmade': 40973,\n",
              " 'ĠTad': 50077,\n",
              " 'ĠMajor': 8386,\n",
              " 'Jose': 23409,\n",
              " 'speed': 12287,\n",
              " 'isations': 38189,\n",
              " 'Ġphony': 41421,\n",
              " 'ĠBauer': 41971,\n",
              " 'ĠPik': 26025,\n",
              " 'dose': 34436,\n",
              " 'Ġominous': 34837,\n",
              " 'Ġcalibration': 36537,\n",
              " 'ĠAngry': 31900,\n",
              " 'With': 3152,\n",
              " 'Ġpull': 2834,\n",
              " 'ĠMord': 29548,\n",
              " 'serious': 34009,\n",
              " 'velt': 18065,\n",
              " 'Ġoptimizations': 41446,\n",
              " 'tu': 28047,\n",
              " 'Ġinclined': 19514,\n",
              " '×ĳ': 49603,\n",
              " 'pire': 5111,\n",
              " 'ĠEarl': 21647,\n",
              " 'BALL': 45463,\n",
              " 'mem': 11883,\n",
              " 'ĠParliament': 8411,\n",
              " 'Ġass': 840,\n",
              " 'ctl': 34168,\n",
              " 'ĠIf': 1002,\n",
              " 'Ġrefres': 17434,\n",
              " 'Operation': 32180,\n",
              " 'Ġmysterious': 11428,\n",
              " 'ĠLund': 33609,\n",
              " 'Ġmagically': 34850,\n",
              " 'Ġalong': 1863,\n",
              " 'Ġtranslate': 15772,\n",
              " 'Ġacre': 31244,\n",
              " 'mop': 35244,\n",
              " 'ĠNX': 42482,\n",
              " 'Ġhazards': 27491,\n",
              " 'Ġbearer': 49508,\n",
              " 'Rec': 6690,\n",
              " 'ĠSect': 24062,\n",
              " 'Ġpicture': 4286,\n",
              " 'Ġvector': 15879,\n",
              " 'Researchers': 25606,\n",
              " 'ried': 2228,\n",
              " 'Ġdistrict': 4783,\n",
              " 'Ġ68': 8257,\n",
              " 'Ġdepletion': 42435,\n",
              " 'JECT': 23680,\n",
              " 'Ġpleasing': 28790,\n",
              " 'mini': 45313,\n",
              " 'ĠCooperative': 43457,\n",
              " 'itional': 1859,\n",
              " 'Ġmiles': 4608,\n",
              " 'ĠBreach': 46148,\n",
              " 'Ġnoticing': 28107,\n",
              " 'ĠYorker': 27980,\n",
              " 'yx': 28391,\n",
              " 'Ġ1440': 49557,\n",
              " 'python': 29412,\n",
              " 'ĠNish': 48438,\n",
              " 'ĠFunctions': 40480,\n",
              " 'ĠEth': 9956,\n",
              " 'Ġwitnesses': 11432,\n",
              " 'Ġdefiance': 35287,\n",
              " 'Ġflatt': 27172,\n",
              " 'ĠIr': 5686,\n",
              " 'Ġpermission': 7170,\n",
              " 'rient': 8289,\n",
              " 'Ġtrending': 29056,\n",
              " 'aceae': 48319,\n",
              " 'ĠMacintosh': 48087,\n",
              " 'Ġuphold': 24963,\n",
              " 'Ġrecognized': 8018,\n",
              " 'ĠBIOS': 27553,\n",
              " 'lang': 17204,\n",
              " 'KNOWN': 44706,\n",
              " 'boxing': 45471,\n",
              " 'ĠFalse': 10352,\n",
              " 'ĠMes': 14937,\n",
              " 'ĠGadget': 39266,\n",
              " 'Ġtor': 7332,\n",
              " 'Ġhemp': 28246,\n",
              " 'ĠTIM': 31742,\n",
              " '781': 49703,\n",
              " 'ĠWent': 29866,\n",
              " 'ĠCollector': 17573,\n",
              " 'Ġmines': 18446,\n",
              " 'as': 292,\n",
              " 'Ġ': 220,\n",
              " 'ial': 498,\n",
              " 'ryption': 13168,\n",
              " 'Ġarrow': 15452,\n",
              " 'ĠFALSE': 26563,\n",
              " 'Ġexclus': 10293,\n",
              " 'Ġcheek': 19353,\n",
              " 'ĠMontana': 18311,\n",
              " 'Ġaff': 1527,\n",
              " 'Load': 8912,\n",
              " 'ims': 12078,\n",
              " 'ĠStur': 26783,\n",
              " 'American': 7437,\n",
              " 'FN': 43221,\n",
              " 'Ġvaries': 17806,\n",
              " 'ĠSantiago': 34802,\n",
              " 'Ġ102': 15143,\n",
              " 'pill': 27215,\n",
              " 'ears': 4127,\n",
              " 'xes': 48169,\n",
              " 'ĠBack': 5157,\n",
              " '82': 6469,\n",
              " 'There': 1858,\n",
              " 'Ġ1968': 15963,\n",
              " 'ĠHollow': 18710,\n",
              " 'ĠSturgeon': 43623,\n",
              " 'agy': 46671,\n",
              " 'orks': 3647,\n",
              " 'Ġbombers': 26127,\n",
              " 'ĠKorea': 4969,\n",
              " 'ĠFriendly': 38683,\n",
              " 'ĠGamma': 43595,\n",
              " 'Ġhabitable': 49055,\n",
              " 'Ġtoll': 13592,\n",
              " 'Ġphenomena': 19428,\n",
              " 'uri': 9900,\n",
              " 'Ġbye': 33847,\n",
              " 'ĠFro': 9734,\n",
              " 'Ġplane': 6614,\n",
              " 'ĠRobot': 16071,\n",
              " 'ĠScient': 9898,\n",
              " 'face': 2550,\n",
              " 'ĠKnicks': 24397,\n",
              " 'helle': 34454,\n",
              " 'camp': 16544,\n",
              " 'ĠRefugees': 38550,\n",
              " 'ĠKirin': 40262,\n",
              " 'Ġarming': 44306,\n",
              " 'mask': 27932,\n",
              " 'EY': 22348,\n",
              " 'iameter': 13173,\n",
              " 'Ġdisciplined': 30654,\n",
              " 'Ġopponents': 7691,\n",
              " 'iframe': 39621,\n",
              " 'ĠShop': 13705,\n",
              " 'ĠDoom': 18688,\n",
              " 'Ġmagnesium': 33525,\n",
              " 'Ġbooze': 39399,\n",
              " 'Ġhunts': 42984,\n",
              " 'evin': 6830,\n",
              " 'Ġquizz': 48258,\n",
              " 'lems': 46367,\n",
              " 'Ġserious': 2726,\n",
              " 'Ġhelmets': 28359,\n",
              " 'BOX': 39758,\n",
              " 'Ġuphe': 40097,\n",
              " 'Ġsilver': 8465,\n",
              " 'Ġoffset': 11677,\n",
              " 'Reports': 37844,\n",
              " 'Ġshows': 2523,\n",
              " 'Ġtypical': 7226,\n",
              " 'Ġorganizer': 26311,\n",
              " '451': 36330,\n",
              " 'ĠRack': 37927,\n",
              " 'Ġcaptured': 7907,\n",
              " 'ĠGuam': 39282,\n",
              " 'Ġfavor': 2661,\n",
              " 'Ult': 16301,\n",
              " ...}"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Creating list of tokens in vocab sorted by their index in vocab**"
      ],
      "metadata": {
        "id": "xFJ0BY2-WKod"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "vocab_list = sorted(tokenizer.vocab.items(), key=lambda x:x[1])"
      ],
      "metadata": {
        "id": "4OkF1IoaKcZq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Verify if the resulting list is sorted by the token indices.**"
      ],
      "metadata": {
        "id": "eI7PnzfwKmoE"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "for k,v in tokenizer.vocab.items():\n",
        "    if v < 10:\n",
        "        print(k, v)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "98syfFzwKk6E",
        "outputId": "f66619c5-5ce4-4a64-de29-953475e42015"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "* 9\n",
            "$ 3\n",
            "% 4\n",
            ") 8\n",
            "& 5\n",
            "! 0\n",
            "\" 1\n",
            "( 7\n",
            "# 2\n",
            "' 6\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "vocab_list[:10]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6XT-UmioKuSt",
        "outputId": "d9094b1a-eb18-46d1-b10e-c00da0fe3614"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('!', 0),\n",
              " ('\"', 1),\n",
              " ('#', 2),\n",
              " ('$', 3),\n",
              " ('%', 4),\n",
              " ('&', 5),\n",
              " (\"'\", 6),\n",
              " ('(', 7),\n",
              " (')', 8),\n",
              " ('*', 9)]"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Save the sorted token labels from vocab as metadata file**"
      ],
      "metadata": {
        "id": "3hkEiSHgLRZ_"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Just create the metadata file\n",
        "\n",
        "with open(os.path.join(log_dir, 'metadata.tsv'), \"w\") as f:\n",
        "    for word, idx in tqdm(vocab_list):\n",
        "        line = str(word.encode(encoding='iso-8859-1', errors='replace'))\n",
        "        line = re.sub(\"^b'\", \"\", line)\n",
        "        line = re.sub('^b\"', \"\", line)\n",
        "        line = re.sub(\"'$\", \"\", line)\n",
        "        line = re.sub('\"$', '', line)\n",
        "        f.write(\"{}\\n\".format(line))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zjv_RY1KKzAo",
        "outputId": "1be29d6e-42c6-4c49-e243-3f62aac0eac0"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "100%|██████████| 50257/50257 [00:00<00:00, 358917.79it/s]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Save the word embeddings**"
      ],
      "metadata": {
        "id": "qw5EGjQpLLsn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "embeddings = tf.Variable(model.transformer.wte.weight.detach().numpy())\n",
        "checkpoint = tf.train.Checkpoint(embedding=embeddings)\n",
        "checkpoint.save(os.path.join(log_dir, \"embedding.ckpt\"))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 36
        },
        "id": "lhPe04ogK-5f",
        "outputId": "ffa29779-257e-48ff-fa99-25978c1cd964"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'./logs/vocab/embedding.ckpt-1'"
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Finally set up tensorboard projector's configuration. This creates a configuration file with .pbtxt extension."
      ],
      "metadata": {
        "id": "zcdLGadQLb5e"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "# Set up config.\n",
        "config = projector.ProjectorConfig()\n",
        "embedding = config.embeddings.add()\n",
        "\n",
        "# The name of the tensor will be suffixed by `/.ATTRIBUTES/VARIABLE_VALUE`.\n",
        "embedding.tensor_name = \"embedding/.ATTRIBUTES/VARIABLE_VALUE\"\n",
        "embedding.metadata_path = 'metadata.tsv'\n",
        "projector.visualize_embeddings(log_dir, config)"
      ],
      "metadata": {
        "id": "hXggPQoyK-2j"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "Run tensorboard to visualize the embeddings. Use UMAP for faster and cleaner visualizations. Search a few keywords and find their nearest neighbours in the 3D space and in the drop down."
      ],
      "metadata": {
        "id": "wbVFM6WwMZPn"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "%load_ext tensorboard\n",
        "%tensorboard --logdir ./logs/vocab/"
      ],
      "metadata": {
        "id": "84o3mB2veDZy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "source": [
        "#### References\n",
        "\n",
        "1. [Tensorboard embedding projector](https://www.tensorflow.org/tensorboard/tensorboard_projector_plugin)\n",
        "\n",
        "2. https://towardsdatascience.com/how-to-visualize-text-embeddings-with-tensorboard-47e07e3a12fb\n",
        "\n",
        "3. https://github.com/huggingface/transformers/issues/1458"
      ],
      "metadata": {
        "id": "_p6vcCYwWOIG"
      }
    }
  ]
}