Joke_Generation.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Joke Generation.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "0e543b812ff9407a97cd42f579fcfa4e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_3e43951ac54b4ba0a996822b8228cce6",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_47d130dfbcbf4c59901f871cc9c2c20a",
              "IPY_MODEL_4b1981dba58a4237bb9a18a18333754c"
            ]
          }
        },
        "3e43951ac54b4ba0a996822b8228cce6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "47d130dfbcbf4c59901f871cc9c2c20a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_85a01e42e2b4409b875e7da04b603dea",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1042301,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1042301,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_78220b1a7c58464c80fab5ba2881324d"
          }
        },
        "4b1981dba58a4237bb9a18a18333754c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_a3853d605a61488c97b0efc8cb86d44e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.04M/1.04M [00:03&lt;00:00, 315kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_adba12ca745844ba8a61f512ba733daf"
          }
        },
        "85a01e42e2b4409b875e7da04b603dea": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "78220b1a7c58464c80fab5ba2881324d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a3853d605a61488c97b0efc8cb86d44e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "adba12ca745844ba8a61f512ba733daf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "b943a1ca56fe455bad60dd7cf776d543": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e77ccda03163479ab006416c6fa281a2",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_a4815015ad1849a2a7f66ab783594206",
              "IPY_MODEL_6ff7375dc1a24c7d9ac5af5b5a2563d2"
            ]
          }
        },
        "e77ccda03163479ab006416c6fa281a2": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a4815015ad1849a2a7f66ab783594206": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4158bc7af22147f0b005f9418963bf6e",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 456318,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 456318,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b02023a55bfa40859602b752fd1326eb"
          }
        },
        "6ff7375dc1a24c7d9ac5af5b5a2563d2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8aec5c9a6cff4474b77386b575a318b1",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 456k/456k [00:00&lt;00:00, 557kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_6f71e6ef11e64dedae5195a16239b4d7"
          }
        },
        "4158bc7af22147f0b005f9418963bf6e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b02023a55bfa40859602b752fd1326eb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8aec5c9a6cff4474b77386b575a318b1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "6f71e6ef11e64dedae5195a16239b4d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0f71498eddae419c95c1bd473b9780ee": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_173dd0e4ed1443cea7e2362b06f07509",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_f04cd57931e34101bed0ad5d50fbb38b",
              "IPY_MODEL_240f0a7b23c94902b8cc066eb00e0ee0"
            ]
          }
        },
        "173dd0e4ed1443cea7e2362b06f07509": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f04cd57931e34101bed0ad5d50fbb38b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_3e17d472f6f94284927b128f1341e7c9",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 718,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 718,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b5e2a8226d2b45b6b860b51ea0aee443"
          }
        },
        "240f0a7b23c94902b8cc066eb00e0ee0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8a18cb2855964a23bb6aeab2bde97ced",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 718/718 [00:30&lt;00:00, 23.2B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a1c23c72760471fb7dad913b5510e18"
          }
        },
        "3e17d472f6f94284927b128f1341e7c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b5e2a8226d2b45b6b860b51ea0aee443": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8a18cb2855964a23bb6aeab2bde97ced": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a1c23c72760471fb7dad913b5510e18": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0fe9a4c85f0949acb092cd00c7db21f3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_ac1e515383d349a2be8afa9f3a40b078",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_71bf0e4f6a2d4d02b084e9ce7397997b",
              "IPY_MODEL_8eaaaa11804a440c869c8f8e1e3e3558"
            ]
          }
        },
        "ac1e515383d349a2be8afa9f3a40b078": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "71bf0e4f6a2d4d02b084e9ce7397997b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_259edb7d4a4e4ef48835444789a83c64",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1419628976,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1419628976,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3461d60ff893418bae490c94c8b099ba"
          }
        },
        "8eaaaa11804a440c869c8f8e1e3e3558": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_88e6ff05f8f44bbdb2722d30ef6b4380",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 1.42G/1.42G [00:29&lt;00:00, 48.3MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a2135dc45d514dc6ac9670f97ff64896"
          }
        },
        "259edb7d4a4e4ef48835444789a83c64": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3461d60ff893418bae490c94c8b099ba": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "88e6ff05f8f44bbdb2722d30ef6b4380": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a2135dc45d514dc6ac9670f97ff64896": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "7oB90qa-JIOK",
        "colab_type": "code",
        "outputId": "b1479fe6-c9ac-4452-8d5d-52cbf7a49563",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "## Quietly installing transformers package to import\n",
        "## the GPT2Tokenizer and TFGPT2LMHeadModel\n",
        "!pip install transformers -q"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\u001b[K     |████████████████████████████████| 675kB 2.7MB/s \n",
            "\u001b[K     |████████████████████████████████| 3.8MB 92kB/s \n",
            "\u001b[K     |████████████████████████████████| 890kB 31.1MB/s \n",
            "\u001b[K     |████████████████████████████████| 1.1MB 38.1MB/s \n",
            "\u001b[?25h  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "0mkkMg9RiJAz",
        "colab_type": "text"
      },
      "source": [
        "## Imports"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1ohArLlf2nj7",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import tensorflow as tf\n",
        "\n",
        "from transformers import GPT2Tokenizer, TFGPT2LMHeadModel\n",
        "\n",
        "import os\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "\n",
        "from tqdm.notebook import tqdm"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qz3BbzQU3nRg",
        "colab_type": "text"
      },
      "source": [
        "## Downloading the Data\n",
        "\n",
        "The dataset is available at https://www.kaggle.com/abhinavmoudgil95/short-jokes/data\n",
        "\n",
        "Sign In to Kaggle and begin the download process for **shortjokes.csv** file. Then copy the link address for the download file and update the _URL varaiable.\n",
        "\n",
        "Once done run all the cells in the notebook. Also, you can cancel the donwload process for the file. :)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "r788uSoz3lkO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "_URL = 'https://storage.googleapis.com/kaggle-data-sets/781%2F1457%2Fcompressed%2Fshortjokes.csv.zip?GoogleAccessId=gcp-kaggle-com@kaggle-161607.iam.gserviceaccount.com&Expires=1591507341&Signature=FyX9Byp7qpIpWTuJi028%2F74JCoCSyi0r8%2FtCGTfx9H0jewKbi%2FGvXRt46owax54aYyDFCfzPmCAWPUKmm%2FKMYgZ%2BqsskQH%2F92PiuQlIT4fttjvKUNEpy14Dcd%2BNy4NCpqLUlU0TCoLgsYEak53yU23QbWBgus1HpFn7UXY1Az8TOjNRBQYk%2FXajaV1qlrrNKRC13K6v6WR2qTsL3tLbzalQWiPxfPv1TwQnqicmYdPxRkhiuv19iX7Y1qpp22ZSzUW6w80e9A5R%2BcAItllI43OrN9HRMljVyJMrDIHP%2FqLAsvTmc2yYBa2muwN4wNDwxOBGbYpsbC0I%2B2F2J1HZbAA%3D%3D'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6N9sG3LID_tn",
        "colab_type": "code",
        "outputId": "def91557-017e-4212-d5b8-4504146ed1ef",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 71
        }
      },
      "source": [
        "path_to_zip = tf.keras.utils.get_file('shortjokes.csv.zip', origin=_URL, extract=True)\n",
        "\n",
        "FILE_PATH = os.path.join(os.path.dirname(path_to_zip), 'shortjokes.csv')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Downloading data from https://storage.googleapis.com/kaggle-data-sets/781%2F1457%2Fcompressed%2Fshortjokes.csv.zip?GoogleAccessId=gcp-kaggle-com@kaggle-161607.iam.gserviceaccount.com&Expires=1591507341&Signature=FyX9Byp7qpIpWTuJi028%2F74JCoCSyi0r8%2FtCGTfx9H0jewKbi%2FGvXRt46owax54aYyDFCfzPmCAWPUKmm%2FKMYgZ%2BqsskQH%2F92PiuQlIT4fttjvKUNEpy14Dcd%2BNy4NCpqLUlU0TCoLgsYEak53yU23QbWBgus1HpFn7UXY1Az8TOjNRBQYk%2FXajaV1qlrrNKRC13K6v6WR2qTsL3tLbzalQWiPxfPv1TwQnqicmYdPxRkhiuv19iX7Y1qpp22ZSzUW6w80e9A5R%2BcAItllI43OrN9HRMljVyJMrDIHP%2FqLAsvTmc2yYBa2muwN4wNDwxOBGbYpsbC0I%2B2F2J1HZbAA%3D%3D\n",
            "10305536/10299687 [==============================] - 0s 0us/step\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "W3xUvbCJFkS1",
        "colab_type": "text"
      },
      "source": [
        "## Preparing the Dataset"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "emWVc6SFRHMk",
        "colab_type": "text"
      },
      "source": [
        "### Extracting jokes list from CSV"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "zCLwNtgSGEYh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "pd.options.display.max_colwidth = None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jgTULP5fEH32",
        "colab_type": "code",
        "outputId": "0a262fe8-4b39-443f-87c7-f3809ca46fd6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "jokes = pd.read_csv(FILE_PATH)\n",
        "jokes.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ID</th>\n",
              "      <th>Joke</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>1</td>\n",
              "      <td>[me narrating a documentary about narrators] \"I can't hear what they're saying cuz I'm talking\"</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>Telling my daughter garlic is good for you. Good immune system and keeps pests away.Ticks, mosquitos, vampires... men.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>I've been going through a really rough period at work this week It's my own fault for swapping my tampax for sand paper.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>4</td>\n",
              "      <td>If I could have dinner with anyone, dead or alive... ...I would choose alive. -B.J. Novak-</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>5</td>\n",
              "      <td>Two guys walk into a bar. The third guy ducks.</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   ID                                                                                                                      Joke\n",
              "0   1                           [me narrating a documentary about narrators] \"I can't hear what they're saying cuz I'm talking\"\n",
              "1   2    Telling my daughter garlic is good for you. Good immune system and keeps pests away.Ticks, mosquitos, vampires... men.\n",
              "2   3  I've been going through a really rough period at work this week It's my own fault for swapping my tampax for sand paper.\n",
              "3   4                                If I could have dinner with anyone, dead or alive... ...I would choose alive. -B.J. Novak-\n",
              "4   5                                                                            Two guys walk into a bar. The third guy ducks."
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ePwTjASsNw50",
        "colab_type": "code",
        "outputId": "f01f7f90-6f84-4412-ee1e-f40e7b861225",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "jokeslist = jokes['Joke'].to_list()\n",
        "jokeslist[:5]"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['[me narrating a documentary about narrators] \"I can\\'t hear what they\\'re saying cuz I\\'m talking\"',\n",
              " 'Telling my daughter garlic is good for you. Good immune system and keeps pests away.Ticks, mosquitos, vampires... men.',\n",
              " \"I've been going through a really rough period at work this week It's my own fault for swapping my tampax for sand paper.\",\n",
              " 'If I could have dinner with anyone, dead or alive... ...I would choose alive. -B.J. Novak-',\n",
              " 'Two guys walk into a bar. The third guy ducks.']"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "u88ehaULRO3U",
        "colab_type": "text"
      },
      "source": [
        "### Creating the Tokenizer for word tokenization"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lPk1pSyjPGid",
        "colab_type": "code",
        "outputId": "e15693a9-f64b-48e1-ef6c-d53b3c7904ab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "0e543b812ff9407a97cd42f579fcfa4e",
            "3e43951ac54b4ba0a996822b8228cce6",
            "47d130dfbcbf4c59901f871cc9c2c20a",
            "4b1981dba58a4237bb9a18a18333754c",
            "85a01e42e2b4409b875e7da04b603dea",
            "78220b1a7c58464c80fab5ba2881324d",
            "a3853d605a61488c97b0efc8cb86d44e",
            "adba12ca745844ba8a61f512ba733daf",
            "b943a1ca56fe455bad60dd7cf776d543",
            "e77ccda03163479ab006416c6fa281a2",
            "a4815015ad1849a2a7f66ab783594206",
            "6ff7375dc1a24c7d9ac5af5b5a2563d2",
            "4158bc7af22147f0b005f9418963bf6e",
            "b02023a55bfa40859602b752fd1326eb",
            "8aec5c9a6cff4474b77386b575a318b1",
            "6f71e6ef11e64dedae5195a16239b4d7"
          ]
        }
      },
      "source": [
        "Tokenizer = GPT2Tokenizer.from_pretrained('gpt2-medium')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0e543b812ff9407a97cd42f579fcfa4e",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1042301.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "b943a1ca56fe455bad60dd7cf776d543",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-O1BnNyNPXRx",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "special_tokens_dict = {'pad_token': 'pad'}\n",
        "num_added_toks = Tokenizer.add_special_tokens(special_tokens_dict)\n",
        "\n",
        "START_TOKEN = '<|start|> '\n",
        "END_TOKEN = ' <|end|>'"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "dv2o6lzfilWm",
        "colab_type": "text"
      },
      "source": [
        "### Create Dataset from List"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jlCWBS-bFt89",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# A utility method to create a tf.data dataset from a List of jokes\n",
        "def jokeslist_to_dataset(jokeslist, tokenizer, \n",
        "                  shuffle=True, batch_size=16, MAX_LEN = 64):\n",
        "  \n",
        "\n",
        "  jokeslist = [START_TOKEN + joke + END_TOKEN for joke in jokeslist]\n",
        "\n",
        "  encodings = [tokenizer.encode_plus(joke,\n",
        "                                  None,\n",
        "                                  add_special_tokens = True,\n",
        "                                  max_length = MAX_LEN,\n",
        "                                  pad_to_max_length = True) \n",
        "              for joke in jokeslist]\n",
        "\n",
        "  ids = [x['input_ids'] for x in encodings]\n",
        "  masks = [x['attention_mask'] for x in encodings]\n",
        "  types = [x['token_type_ids'] for x in encodings]\n",
        "\n",
        "  inputs = {}\n",
        "  inputs['input_ids'] = ids\n",
        "  inputs['attention_mask'] = masks\n",
        "  inputs['token_type_ids'] = types\n",
        "\n",
        "  ds = tf.data.Dataset.from_tensor_slices(inputs)\n",
        "\n",
        "  if shuffle:\n",
        "    ds = ds.shuffle(buffer_size=len(jokeslist))\n",
        "\n",
        "  ds = ds.batch(batch_size)\n",
        "\n",
        "  return ds"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NhgJUfaKir_t",
        "colab_type": "text"
      },
      "source": [
        "Note: This is a costly process since all the tokenization is done immediately. Thus it is expected to be slow. The only advantage is since everything is processed and kept in memory we are saving repated operationg while training."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6RpJpWbVOEsZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "## In case you wish to only test the model do not run this cell\n",
        "\n",
        "jokes_dataset = jokeslist_to_dataset(jokeslist, Tokenizer)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LzdrkAGI88Rc",
        "colab_type": "code",
        "outputId": "fd097f5d-bb1e-457f-886d-81eddc303461",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        }
      },
      "source": [
        "for x in jokes_dataset:\n",
        "  a, b, c = x['input_ids'], x['attention_mask'], x['token_type_ids']\n",
        "  print(a[0], b[0], c[0])\n",
        "  break"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "tf.Tensor(\n",
            "[   27    91  9688    91    29   314  1549   787   257  1964  9707   475\n",
            "   340   561   655   886   510   852  7018    13  1279    91   437    91\n",
            "    29 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636\n",
            " 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636\n",
            " 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636 15636\n",
            " 15636 15636 15636 15636], shape=(64,), dtype=int32) tf.Tensor(\n",
            "[1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], shape=(64,), dtype=int32) tf.Tensor(\n",
            "[0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0\n",
            " 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0 0], shape=(64,), dtype=int32)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "YMiORQ8iXYNK",
        "colab_type": "text"
      },
      "source": [
        "## The Model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MFGKP90jTVn1",
        "colab_type": "code",
        "outputId": "46a33044-4f78-4546-b923-2e996b652096",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 115,
          "referenced_widgets": [
            "0f71498eddae419c95c1bd473b9780ee",
            "173dd0e4ed1443cea7e2362b06f07509",
            "f04cd57931e34101bed0ad5d50fbb38b",
            "240f0a7b23c94902b8cc066eb00e0ee0",
            "3e17d472f6f94284927b128f1341e7c9",
            "b5e2a8226d2b45b6b860b51ea0aee443",
            "8a18cb2855964a23bb6aeab2bde97ced",
            "3a1c23c72760471fb7dad913b5510e18",
            "0fe9a4c85f0949acb092cd00c7db21f3",
            "ac1e515383d349a2be8afa9f3a40b078",
            "71bf0e4f6a2d4d02b084e9ce7397997b",
            "8eaaaa11804a440c869c8f8e1e3e3558",
            "259edb7d4a4e4ef48835444789a83c64",
            "3461d60ff893418bae490c94c8b099ba",
            "88e6ff05f8f44bbdb2722d30ef6b4380",
            "a2135dc45d514dc6ac9670f97ff64896"
          ]
        }
      },
      "source": [
        "model = TFGPT2LMHeadModel.from_pretrained('gpt2-medium')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0f71498eddae419c95c1bd473b9780ee",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=718.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "0fe9a4c85f0949acb092cd00c7db21f3",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1419628976.0, style=ProgressStyle(descr…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E4v3XN3QXfPT",
        "colab_type": "code",
        "outputId": "4efc966f-12f5-4096-ae4e-2b141a1b9ef7",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 187
        }
      },
      "source": [
        "model.summary()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Model: \"tfgp_t2lm_head_model\"\n",
            "_________________________________________________________________\n",
            "Layer (type)                 Output Shape              Param #   \n",
            "=================================================================\n",
            "transformer (TFGPT2MainLayer multiple                  354823168 \n",
            "=================================================================\n",
            "Total params: 354,823,168\n",
            "Trainable params: 354,823,168\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Ha-i3KzwiAV2",
        "colab_type": "text"
      },
      "source": [
        "## Loss Function and Optimizer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wBlnY-kOEQoq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "loss_function = tf.losses.SparseCategoricalCrossentropy(from_logits=True)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ekhYm_pBXheT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "optimizer = tf.keras.optimizers.Adam(learning_rate=1e-5, epsilon=1e-08, clipnorm=1.0)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6nU_9GzviDuL",
        "colab_type": "text"
      },
      "source": [
        "## Checkpointing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kXlXQJSJnBYH",
        "colab_type": "code",
        "outputId": "6a6b96a5-3da3-4a31-9fbe-66bfa8a4622f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly\n",
            "\n",
            "Enter your authorization code:\n",
            "··········\n",
            "Mounted at /content/gdrive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "VrCpGfDdnN-b",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "CHECKPOINT_PATH = \"/content/gdrive/My Drive/Weights/JokeGenGPT2\""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JWoYstRaRSTa",
        "colab_type": "code",
        "outputId": "d3f5e832-1100-4826-f0e5-0e610d2af664",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "checkpoint_path = CHECKPOINT_PATH\n",
        "\n",
        "ckpt = tf.train.Checkpoint(model = model)\n",
        "\n",
        "ckpt_manager = tf.train.CheckpointManager(ckpt, checkpoint_path, max_to_keep=5)\n",
        "\n",
        "# if a checkpoint exists, restore the latest checkpoint.\n",
        "if ckpt_manager.latest_checkpoint:\n",
        "  ckpt.restore(ckpt_manager.latest_checkpoint)\n",
        "  print ('Latest checkpoint restored!!')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Latest checkpoint restored!!\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3wv-oeml7g89",
        "colab_type": "text"
      },
      "source": [
        "## Training"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P0L1ZZT2RLrT",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "@tf.function\n",
        "def train_step(data_dict):  \n",
        "  with tf.GradientTape() as tape:\n",
        "\n",
        "    outputs = model(data_dict)\n",
        "\n",
        "    lm_logits = outputs[0]\n",
        "    labels = data_dict['input_ids']\n",
        "\n",
        "    ## For computing loss we remove the last element from logits and \n",
        "    ## first from labels. Thus we need the model to learn to predict next\n",
        "    ## word more confidently among the others. \n",
        "    shift_logits = lm_logits[..., :-1, :]\n",
        "    shift_labels = labels[..., 1:]\n",
        "    \n",
        "    loss = loss_function(tf.reshape(shift_labels, (-1,)),\n",
        "         tf.reshape(shift_logits, \n",
        "                   (-1, shift_logits.shape[-1])))\n",
        "\n",
        "  gradients = tape.gradient(loss, model.trainable_variables)    \n",
        "  optimizer.apply_gradients(zip(gradients, model.trainable_variables))\n",
        "\n",
        "  return loss"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "TdfNtCJYzKzz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EPOCHS = 4\n",
        "\n",
        "for epoch in range(EPOCHS):\n",
        "  \n",
        "  for batch, data in tqdm(enumerate(jokes_dataset)):\n",
        "    loss = train_step(data)\n",
        "    if batch % 100 == 0:\n",
        "      print('Epoch : {0} Batch : {1} ---- Loss : {2}'.format(epoch+1, batch+1, loss))\n",
        "\n",
        "  \n",
        "\n",
        "  ckpt_save_path = ckpt_manager.save()\n",
        "  print ('Saving checkpoint for epoch {} at {}'.format(epoch+1,\n",
        "                                                        ckpt_save_path))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Qpyh-ydg6n2a",
        "colab_type": "text"
      },
      "source": [
        "## Inference"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Wpd2PoX57peW",
        "colab_type": "text"
      },
      "source": [
        "### Generating Joke from Scratch"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "P3QRDWIi9xLJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def exploit_best_token_while_exploring(probabilites, exploration_len=5):\n",
        "    ## Get the top k probabilites indices where k is exploration_len \n",
        "    top_indices = np.argpartition(probabilites, -exploration_len)\\\n",
        "                            [-exploration_len:]\n",
        "\n",
        "    ## Getting top proabilities value\n",
        "    top_probabilities = probabilites[top_indices]\n",
        "\n",
        "    ## Normalizing it so that they sum is 1\n",
        "    top_probabilities = top_probabilities / np.sum(top_probabilities)\n",
        "\n",
        "    ## Although we choose next token randomly for our options we \n",
        "    ## pass the probabilities associated with each to account for the\n",
        "    ## model's confidence for the token also. \n",
        "    choice = np.random.choice(exploration_len, 1, p = top_probabilities)\n",
        "\n",
        "    ## Return next token id based on choice\n",
        "    next_token_id = int(top_indices[choice][0])\n",
        "    return next_token_id"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "moB2eE9S-34S",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def generate_joke(joke_length = 64):\n",
        "\n",
        "  ## Begin by appending the START_TOKEN to our current joke\n",
        "  current_joke = tf.expand_dims(tf.convert_to_tensor(Tokenizer.encode(START_TOKEN)), 0)\n",
        "  \n",
        "  for pos in range(joke_length):\n",
        "\n",
        "    ## Get output of model for the current_joke\n",
        "    output = model(current_joke)\n",
        "\n",
        "    ## Getting the logits value from output tuple i.e. (logits)\n",
        "    logits = output[0]\n",
        "\n",
        "    ## Logits is of the shape (BATCH, LEN_INPUT) \n",
        "    ## So, since batch size is 1 we get 0th index\n",
        "    ## and the softmax for only the next possible word i.e. -1\n",
        "    softmax_logits = tf.nn.softmax(logits[0, -1], axis=0).numpy()\n",
        "\n",
        "    ## Depending on whether we are in initial or final stages of joke\n",
        "    ## we determine how many options we should explore to make sure we have\n",
        "    ## variety in jokes generated\n",
        "    if pos == 0:\n",
        "      # If we are predicting first word we need maximum exploration\n",
        "      exploration_len = 50\n",
        "\n",
        "    elif pos < 4:\n",
        "      # The next three words have exploration length 15\n",
        "      exploration_len = 15\n",
        "\n",
        "    else:\n",
        "      # As we move further we narrow our exploration length\n",
        "      exploration_len = 4\n",
        "\n",
        "    ## Get the token we should append to current joke\n",
        "    token_to_append = exploit_best_token_while_exploring(softmax_logits, \n",
        "                                                         exploration_len)\n",
        "\n",
        "    ## Append the token to current joke\n",
        "    current_joke = tf.concat([current_joke, \n",
        "                              tf.ones((1,1), dtype = tf.int32)*token_to_append], \n",
        "                             axis = 1)\n",
        "    \n",
        "    ## In case the token belongs to the END_TOKEN we return it as complete joke\n",
        "    if token_to_append in Tokenizer.encode(END_TOKEN):\n",
        "      return Tokenizer.decode(list(tf.squeeze(current_joke).numpy()))\n",
        "  \n",
        "  ## If we did not get end token it means no joke is formed\n",
        "  return None"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMEKPeH5EEKi",
        "colab_type": "code",
        "outputId": "6089955e-16c6-4ce9-a199-71b8530a988d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "generate_joke()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<|start|> I had a dream last night that I was the president of the US... I had to sleep through it, but it was good. <'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V4sbOT0HEa8O",
        "colab_type": "code",
        "outputId": "aff3b11b-2ffa-4719-b02b-57c0f3ebb787",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "generate_joke()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<|start|> \"What\\'s a good joke about the Titanic?\" \"The one about the icebergs?\" \"No, the iceberg jokes aren\\'t good. The Titanic\\'s a good example.\" <'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 83
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PLALyIYTLxQV",
        "colab_type": "code",
        "outputId": "e29d7aaf-ea24-45cf-fc14-c4d0a8367b31",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "generate_joke()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"<|start|> Why does Peter Pan always fly? It's because he Neverlands. <\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 89
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oqNdESVlGb2w",
        "colab_type": "code",
        "outputId": "256489d9-8b94-49ae-e687-52e5ead49620",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "generate_joke()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<|start|> Q: What kind of shoes do pirates wear? A: Sneakers <'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 47
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DizJsii0rvlW",
        "colab_type": "code",
        "outputId": "96a2a3b3-505e-4cfc-9f41-ea0336ff1978",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "generate_joke()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"<|start|> Why did the blonde have a sore throat? Because she couldn't concentrate <\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 50
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lH84XzqAyyEJ",
        "colab_type": "code",
        "outputId": "45560790-2631-4e94-a6b0-0c0bf3a32da0",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "generate_joke()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<|start|> I was asked to be the spokesperson for the LGBT community in the United States. I said, \"I\\'ll be gay.\" They laughed and I said, \"I\\'m sorry, I don\\'t think I\\'m gay.\" <'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "daXhybek7uLd",
        "colab_type": "text"
      },
      "source": [
        "### Completing Sentences in Humorous Way"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4tj_bTMJ0DIq",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def complete_joke(initial_string = '', joke_length = 64):\n",
        "\n",
        "  ## Begin by appending the START_TOKEN along with initial string to \n",
        "  ## our current joke sentence\n",
        "  current_joke = tf.expand_dims(\n",
        "      tf.convert_to_tensor(Tokenizer.encode(START_TOKEN + initial_string)), \n",
        "      0)\n",
        "  \n",
        "  for pos in range(joke_length):\n",
        "\n",
        "    ## Get output of model for the current_joke\n",
        "    output = model(current_joke)\n",
        "\n",
        "    ## Getting the logits value from output tuple i.e. (logits)\n",
        "    logits = output[0]\n",
        "\n",
        "    ## Logits is of the shape (BATCH, LEN_INPUT) \n",
        "    ## So, since batch size is 1 we get 0th index\n",
        "    ## and the softmax for only the next possible word i.e. -1\n",
        "    softmax_logits = tf.nn.softmax(logits[0, -1], axis=0).numpy()\n",
        "\n",
        "    ## Depending on whether we are in initial or final stages of joke\n",
        "    ## we determine how many options we should explore to make sure we have\n",
        "    ## variety in jokes generated\n",
        "    if pos == 0:\n",
        "      # If we are predicting first word we need maximum exploration\n",
        "      exploration_len = 50\n",
        "\n",
        "    elif pos < 4:\n",
        "      # The next three words have exploration length 15\n",
        "      exploration_len = 15\n",
        "\n",
        "    else:\n",
        "      # As we move further we narrow our exploration length\n",
        "      exploration_len = 4\n",
        "\n",
        "    ## Get the token we should append to current joke\n",
        "    token_to_append = exploit_best_token_while_exploring(softmax_logits, \n",
        "                                                         exploration_len)\n",
        "\n",
        "    ## Append the token to current joke\n",
        "    current_joke = tf.concat([current_joke, \n",
        "                              tf.ones((1,1), dtype = tf.int32)*token_to_append], \n",
        "                             axis = 1)\n",
        "    \n",
        "    ## In case the token belongs to the END_TOKEN we return it as complete joke\n",
        "    if token_to_append in Tokenizer.encode(END_TOKEN):\n",
        "      return Tokenizer.decode(list(tf.squeeze(current_joke).numpy()))\n",
        "  \n",
        "  ## If we did not get end token it means no joke is formed and we try again\n",
        "  return complete_joke(initial_string, joke_length)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YTdnCEZAUhZD",
        "colab_type": "code",
        "outputId": "bd0bad15-a6bd-4d75-c84f-1fc7887fae67",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "complete_joke(\"Jokes are\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"<|start|> Jokes are just jokes. If i ever had a son i'd call him 'Joke' <\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 178
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nu0W9E4vSx1D",
        "colab_type": "code",
        "outputId": "262b1f6b-67f1-4a74-fb25-8cb2a0c0cacd",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "complete_joke(\"I am trained enough\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<|start|> I am trained enough to recognize a fake name on a Facebook profile. <'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 173
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Y9HE08nE8BM9",
        "colab_type": "code",
        "outputId": "514b7b9f-a2d8-45a3-9f22-f08dff9da107",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "complete_joke(\"I can do this all day\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "'<|start|> I can do this all day long. \"I can do this all day long.\" -People with no idea what the word \"all\" means <'"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 137
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BKLzVnBp8JyZ",
        "colab_type": "code",
        "outputId": "03cf9241-a821-4536-d3c5-22edbe55b09f",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "complete_joke(\"Hope is a good thing\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"<|start|> Hope is a good thing.. until you don't have any <\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 182
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "kzjvvf0Q-uI3",
        "colab_type": "code",
        "outputId": "030408b9-8d49-4880-dc96-c883c2e59bdb",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "complete_joke(\"Where is \")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"<|start|> Where is a good place for a kid's birthday party? In a coffin. <\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 36
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cmNJgUW5-z0F",
        "colab_type": "code",
        "outputId": "b172b142-2a28-447d-ce8d-989059321c72",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "complete_joke(\"Why do\")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"<|start|> Why do girls always get lost in the woods? Because they're not smart enough to walk alone. <\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 38
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_rMELKCc-7w5",
        "colab_type": "code",
        "outputId": "31ba34f4-1774-4dc8-af69-1b35856ae1ba",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "complete_joke(\"Don't lie \")"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"<|start|> Don't lie to your spouse, your boss, your neighbor, or anyone else about how many people you've slept with. <\""
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 45
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3qwBU2J1MCQh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}