question_answering_(klue_mrc)_BERT.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "question answering (klue_mrc)-BERT",
      "provenance": [],
      "collapsed_sections": [],
      "authorship_tag": "ABX9TyMvwkdLiXy4RfZ6L3aJWpAm",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "08baf1a74a2c4cebbb5dd83030e25c69": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_a95ecb9980f14ac691d4e30fe2036d9b",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_86d8497adcb640c8ba7dc961fbf806ec",
              "IPY_MODEL_0df74cb5fe494f2faf8c7c57c7220ab2",
              "IPY_MODEL_43524da14fcf42f9a92740eda8e3078b"
            ]
          }
        },
        "a95ecb9980f14ac691d4e30fe2036d9b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "86d8497adcb640c8ba7dc961fbf806ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1879c3bb390943849ef9ca934ca20a99",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7240ca258f204945952347c32c37c749"
          }
        },
        "0df74cb5fe494f2faf8c7c57c7220ab2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_e5f00a84e3f84abfbb81253c9322bb9f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 5,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 5,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_94f3779636ac4301b83614cfe0cbb673"
          }
        },
        "43524da14fcf42f9a92740eda8e3078b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_d7ca52c582584303aa8f455d98589181",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 5/5 [00:33&lt;00:00,  5.35s/ba]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_008e655d6f61432f86cb1d5502fb74a1"
          }
        },
        "1879c3bb390943849ef9ca934ca20a99": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7240ca258f204945952347c32c37c749": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e5f00a84e3f84abfbb81253c9322bb9f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "94f3779636ac4301b83614cfe0cbb673": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d7ca52c582584303aa8f455d98589181": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "008e655d6f61432f86cb1d5502fb74a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1436af3e6fe347d1b067ef729e7dedc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_32ace2b72e594204994fae0d3e465bc7",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_c2d1731c3ce54d30ae9e88d6643688d3",
              "IPY_MODEL_0fcfa7dd647e4301a635429de76af648",
              "IPY_MODEL_53d14a74367440269353e7e9778ee85a"
            ]
          }
        },
        "32ace2b72e594204994fae0d3e465bc7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "c2d1731c3ce54d30ae9e88d6643688d3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_56c75e6326ec4601906c687dbc96c341",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "100%",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2708e34134e14c0ca3d87c7b2c056007"
          }
        },
        "0fcfa7dd647e4301a635429de76af648": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_6f5e85f6903e48d994eaafb9263abb0c",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 4008,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 4008,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3a0f87b9409f45c89af5fb6265244f09"
          }
        },
        "53d14a74367440269353e7e9778ee85a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_1661341a9dac418ab998b1dffccab0b3",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4008/4008 [00:21&lt;00:00, 180.79it/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_2bcda5cf72404e0cb28d790c639c4e29"
          }
        },
        "56c75e6326ec4601906c687dbc96c341": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2708e34134e14c0ca3d87c7b2c056007": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "6f5e85f6903e48d994eaafb9263abb0c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3a0f87b9409f45c89af5fb6265244f09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1661341a9dac418ab998b1dffccab0b3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "2bcda5cf72404e0cb28d790c639c4e29": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "4f4b214805db4b1bba7855293477ba96": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_c2d5527a92c24c68a3b3bb1ee5f4d5bb",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0d8620a2cd1f4543981dc5e05bd48c80",
              "IPY_MODEL_16bdc720b44043b58cddccea79eb5a19",
              "IPY_MODEL_9e5286cef5af44229ef52fd637712b3e"
            ]
          }
        },
        "c2d5527a92c24c68a3b3bb1ee5f4d5bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0d8620a2cd1f4543981dc5e05bd48c80": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9e4a8203b5f64be3bc98d960e94efb0f",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: ",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d5496e20c1e94c80910b74f2e7e5a00b"
          }
        },
        "16bdc720b44043b58cddccea79eb5a19": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_a8d91772e23842099a27c0065ee6b5f4",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1726,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1726,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_a549b79c10ec49f4b01cc4ce81fcea92"
          }
        },
        "9e5286cef5af44229ef52fd637712b3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_51140dd903494f16a3cb1f22a0ce9092",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 4.51k/? [00:00&lt;00:00, 121kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_8f8c3bbe1ca243939b216096e1845a54"
          }
        },
        "9e4a8203b5f64be3bc98d960e94efb0f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d5496e20c1e94c80910b74f2e7e5a00b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a8d91772e23842099a27c0065ee6b5f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "a549b79c10ec49f4b01cc4ce81fcea92": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "51140dd903494f16a3cb1f22a0ce9092": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "8f8c3bbe1ca243939b216096e1845a54": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "d8a558407dad46969d40a33088eb3121": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_55c63e2653144dfa9eb73116ca70b5f9",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_2b31942058ac4a6bb20a436828407164",
              "IPY_MODEL_7bda3b5b23e141ddb416e6521ce7de51",
              "IPY_MODEL_77f6db4122174db990c368ce2ca964cc"
            ]
          }
        },
        "55c63e2653144dfa9eb73116ca70b5f9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2b31942058ac4a6bb20a436828407164": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_bb255d7240724fe88fabd18c1edffbb7",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": "Downloading: ",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b7a1d4b8848e4c15b6cf041e7d4370dc"
          }
        },
        "7bda3b5b23e141ddb416e6521ce7de51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_1c863110097e45cb9512876c0fed9598",
            "_dom_classes": [],
            "description": "",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 1119,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 1119,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5b6562ba44e949a2904bda8c17ea5630"
          }
        },
        "77f6db4122174db990c368ce2ca964cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_9bb5ff8b97c445d7ae7573ce234bfece",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 3.31k/? [00:00&lt;00:00, 98.1kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_7420737a61ea452684d35a4c9268152d"
          }
        },
        "bb255d7240724fe88fabd18c1edffbb7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b7a1d4b8848e4c15b6cf041e7d4370dc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "1c863110097e45cb9512876c0fed9598": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5b6562ba44e949a2904bda8c17ea5630": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9bb5ff8b97c445d7ae7573ce234bfece": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "7420737a61ea452684d35a4c9268152d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/seopbo/nlp_tutorials/blob/main/question_answering_(klue_mrc)_BERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "8sJoxEU0fJ5f"
      },
      "source": [
        "# Question answering - BERT\n",
        "- pre-trained language model로는 `klue/bert-base`를 사용합니다.\n",
        "  - https://huggingface.co/klue/bert-base\n",
        "- extractive question asnwering을 수행하는 예시 데이터셋으로는 klue의 mrc를 사용합니다.\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Z8-sxczRgiNj"
      },
      "source": [
        "## Setup\n",
        "어떠한 GPU가 할당되었는 지 아래의 코드 셀을 실행함으로써 확인할 수 있습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "bbvZuVIdgim6",
        "outputId": "71670f1d-5fbb-4ce4-e7af-0025cfacc274"
      },
      "source": [
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "\n",
        "if gpu_info.find('failed') >= 0:\n",
        "    print('Not connected to a GPU')\n",
        "else:\n",
        "    print(gpu_info)"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tue Dec 28 01:32:55 2021       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 495.44       Driver Version: 460.32.03    CUDA Version: 11.2     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla P100-PCIE...  Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   43C    P0    38W / 250W |      0MiB / 16280MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qA_Guaj4g-ME"
      },
      "source": [
        "아래의 코드 셀을 실행함으로써 본 노트북을 실행하기위한 library를 install하고 load합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "D3Djk64xhAPw",
        "outputId": "a59a6018-e0d6-4a42-97f0-64ccdb46524d"
      },
      "source": [
        "!pip install torch\n",
        "!pip install transformers\n",
        "!pip install datasets\n",
        "\n",
        "from pprint import pprint\n",
        "import torch\n",
        "import transformers\n",
        "import datasets"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.10.0+cu111)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.10.0.2)\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.15.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: tokenizers<0.11,>=0.10.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.10.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.62.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers) (0.0.46)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.4.0)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (0.2.1)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.8.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (3.10.0.2)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.6)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.6.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2021.10.8)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.1.0)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (1.17.0)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2.23.0)\n",
            "Requirement already satisfied: dill in /usr/local/lib/python3.7/dist-packages (from datasets) (0.3.4)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.7/dist-packages (from datasets) (4.62.3)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.7/dist-packages (from datasets) (2.0.2)\n",
            "Requirement already satisfied: pyarrow!=4.0.0,>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (3.0.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from datasets) (1.19.5)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.7/dist-packages (from datasets) (3.8.1)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.7/dist-packages (from datasets) (0.70.12.2)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.7/dist-packages (from datasets) (1.1.5)\n",
            "Requirement already satisfied: fsspec[http]>=2021.05.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (2021.11.1)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from datasets) (4.8.2)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.1.0 in /usr/local/lib/python3.7/dist-packages (from datasets) (0.2.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.10.0.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (3.4.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0.0,>=0.1.0->datasets) (6.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->datasets) (3.0.6)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2021.10.8)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.19.0->datasets) (1.24.3)\n",
            "Requirement already satisfied: asynctest==0.13.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (0.13.0)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.7.2)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (1.2.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (5.2.0)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (2.0.8)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (21.2.0)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.7/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->datasets) (3.6.0)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas->datasets) (2018.9)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v8uwFSCVhTlv"
      },
      "source": [
        "## Preprocess data\n",
        "1. `klue/bert-base`가 사용한 subword tokenizer를 load합니다.\n",
        "2. `datasets` library를 이용하여 klue의 mrc를 load합니다.\n",
        "3. 1의 subword tokenizer를 이용 klue mrc의 data를 span detection을 수행할 수 있는 형태, train example로 transformation합니다.\n",
        "\n",
        "- `[CLS] question_tokens [SEP] context_tokens [SEP]`로 만들고, 이를 list_of_integers로 transform합니다.\n",
        "- 단 주어진 model의 `max_sequence_length` list_of_integers를 처리할 수 있는 길이가 아닐 경우에 아래의 조치를 수행합니다.\n",
        "  - `question_tokens`는 상대적으로 짧고 `context_tokens`는 상대적으로 길이가 길 것이기 때문에 truncation이 발생할 경우, `context_tokens`만 truncation이 이루어지도록 합니다 tokenizer의 `__call__` method 활용 시, `truncation` parameter에 `\"only_second\"`를 전달하여 위와 같은 동작을 수행할 수 있습니다.\n",
        "  - 위의 동작을 수행 시 question에 대한 답이 잘려 나갈 가능성이 있습니다. tokenizer의 `__call__` method 활용 시, `stride` parameter에 적절한 값을 주어 `context_tokens`를 겹쳐보게 만들고, `return_overflowing_tokens`에 `True`를 전달하여, 답이 잘려 나갈 가능성을 방지할 수 있습니다. 단 이 경우 하나의 (question, context)에서 여러개의 training example이 생성될 수 있습니다.\n",
        "\n",
        "- answer가 context에서 어디에 위치하는 지 확인합니다. 특히 우리가 사용하는 tokenizer의 결과에 맞추어 tokenizer의 결과에서 몇 개의 연속된 token이 answer와 일치하는 지에 대한 정보를 만들어주어야합니다. tokenizer의 `__call__` method 활용 시, `return_offsets_mapping` parameter에 `True`를 전달하여 얻은 offset 정보로 위의 정보를 생성해야합니다.\n"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ELOFARIx63xt",
        "outputId": "547dc749-f679-42dd-ae02-d4fe9ff55623"
      },
      "source": [
        "from transformers import AutoTokenizer, AutoConfig\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"klue/bert-base\")\n",
        "\n",
        "print(tokenizer.__class__)"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'transformers.models.bert.tokenization_bert_fast.BertTokenizerFast'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "noK16hrChYyH",
        "outputId": "c2833fe0-9978-4b3d-f0a3-005dd27fc696"
      },
      "source": [
        "from datasets import load_dataset\n",
        "\n",
        "cs = load_dataset(\"klue\", \"mrc\", split=\"train\")\n",
        "# 대답할 수 없는 경우는 제외\n",
        "cs = cs.filter(lambda example: example[\"is_impossible\"] == False)\n",
        "cs = cs.train_test_split(0.1)\n",
        "\n",
        "train_cs = cs[\"train\"]\n",
        "valid_cs = cs[\"test\"]\n",
        "\n",
        "test_cs = load_dataset(\"klue\", \"mrc\", split=\"validation\")\n",
        "test_cs = test_cs.filter(lambda example: example[\"is_impossible\"] == False)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Reusing dataset klue (/root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e)\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e/cache-8775884b4f525c7f.arrow\n",
            "Loading cached split indices for dataset at /root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e/cache-a30f2529f0b3c2c2.arrow and /root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e/cache-58bb360da3c80e7c.arrow\n",
            "Reusing dataset klue (/root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e)\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e/cache-4cfb9e85dc37f419.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Sq4IZ9gNITld"
      },
      "source": [
        "위에서 말한 내용을 실제로 동작시켜보며 알아보면 아래와 같습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Dey9sW5KIXaW",
        "outputId": "8e2f2571-f36c-41b2-a043-49989565f6b9"
      },
      "source": [
        "example = train_cs[0]\n",
        "pprint(example)"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'answers': {'answer_start': [12], 'text': ['류재선']},\n",
            " 'context': '한국전기공사협회(회장 류재선)와 전기공사공제조합(이사장 김성관)은 에너지시민연대와 공동으로 7월 16일 서울 명동에서 '\n",
            "            '‘에너지 절약 캠페인’을 실시했다. 이번 캠페인은 명동 일대 상점과 시민들에게 안전하고 효율적인 전기사용의 중요성을 '\n",
            "            '알리고 에너지 절약의 필요성을 전파하기 위한 것으로, 전력산업계를 대표하는 협회, 조합, 전기신문사, 전기산업연구원, '\n",
            "            '전기공사공제조합장학회, 엘비라이프 등 각 기관 임직원과 대학생 자원봉사자 200여명이 함께해 행사의 의미를 더했다. '\n",
            "            '참가자들은 행사 시작에 앞서 전기절약의 중요성을 담은 퍼포먼스를 펼친 후, 명동 일대를 행진하며 시민들이 직접 실천할 '\n",
            "            '수 있는 내용이 담긴 부채와 쿨 스카프를 전파했다. 또 각 상점을 방문해 안전하고 효율적인 전기기기 사용법을 전파하며 '\n",
            "            '에너지 절약 실천을 당부했다. 이날 전기공사협회 류재선 회장은 “올바른 전기 사용의 중요성이 많이 홍보됐지만 아직 '\n",
            "            '위험하고 비효율적으로 사용되고 있다”며 시민들의 관심과 전기 절약 노력을 당부했다. 이어 김성관 전기공사공제조합 '\n",
            "            '이사장은 “안정적인 전력 공급을 위해 전력사용량이 급증하는 여름철 효율적인 냉방 등으로 에너지절약을 실천해 주시길 '\n",
            "            '바란다”고 말했다. 한국전기공사협회는 이번 캠페인이 안정적인 전력공급을 위한 에너지절약에 전기건설산업계가 앞장섰다는 '\n",
            "            '점에서 의미가 있다고 밝혔다.',\n",
            " 'guid': 'klue-mrc-v1_train_15747',\n",
            " 'is_impossible': False,\n",
            " 'news_category': '산업',\n",
            " 'question': '캠페인의 의의를 언급한 단체의 대표는?',\n",
            " 'question_type': 2,\n",
            " 'source': 'acrofan',\n",
            " 'title': '한국전기공사협회, 명동 일대서 ‘에너지 절약 캠페인’ 개최'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "9WZLDSjOIMbk",
        "outputId": "81f7428c-e129-4ede-eade-cdb236f65233"
      },
      "source": [
        "# truncation parameter에 따른 차이\n",
        "print(tokenizer.convert_ids_to_tokens(tokenizer(example[\"question\"], example[\"context\"], truncation=True, max_length=24)[\"input_ids\"]))\n",
        "print(tokenizer.convert_ids_to_tokens(tokenizer(example[\"question\"], example[\"context\"], truncation=\"only_second\", max_length=24)[\"input_ids\"]))"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '[SEP]', '한국', '##전기', '##공사', '##협회', '(', '회장', '류', '##재', '##선', ')', '와', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '한국', '##전기', '##공사', '##협회', '(', '회장', '류', '##재', '##선', ')', '[SEP]']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rbgfvMgJK1uE",
        "outputId": "2db8de74-f78c-4948-dd82-6ce513b74d22"
      },
      "source": [
        "# return_overflowing_tokens=True의 결과\n",
        "tokenized_example = tokenizer(example[\"question\"], example[\"context\"], truncation=\"only_second\", max_length=24, return_overflowing_tokens=True)\n",
        "\n",
        "for input_ids in tokenized_example[\"input_ids\"]:\n",
        "    print(tokenizer.convert_ids_to_tokens(input_ids))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '한국', '##전기', '##공사', '##협회', '(', '회장', '류', '##재', '##선', ')', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '와', '전기', '##공사', '##공', '##제', '##조합', '(', '이사장', '김성', '##관', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', ')', '은', '에너지', '##시', '##민', '##연대', '##와', '공동', '##으로', '7', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##월', '16', '##일', '서울', '명동', '##에서', '‘', '에너지', '절약', '캠페인', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '’', '을', '실시', '##했', '##다', '.', '이번', '캠페인', '##은', '명동', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '일대', '상점', '##과', '시민', '##들', '##에', '##게', '안전', '##하고', '효율', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##적인', '전기', '##사', '##용', '##의', '중요', '##성', '##을', '알리', '##고', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '에너지', '절약', '##의', '필요', '##성', '##을', '전파', '##하기', '위한', '것', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##으로', ',', '전력', '##산업', '##계', '##를', '대표', '##하', '##는', '협회', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', ',', '조합', ',', '전기', '##신문', '##사', ',', '전기', '##산업', '##연구원', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', ',', '전기', '##공사', '##공', '##제', '##조합', '##장', '##학', '##회', ',', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '엘', '##비', '##라이', '##프', '등', '각', '기관', '임직원', '##과', '대학생', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '자원', '##봉사', '##자', '200', '##여', '##명', '##이', '함께', '##해', '행사', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##의', '의미', '##를', '더', '##했', '##다', '.', '참가자', '##들', '##은', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '행사', '시작', '##에', '앞서', '전기', '##절', '##약', '##의', '중요', '##성', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##을', '담', '##은', '퍼포먼스', '##를', '펼친', '후', ',', '명동', '일대', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##를', '행진', '##하', '##며', '시민', '##들이', '직접', '실천', '##할', '수', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '있', '##는', '내용', '##이', '담긴', '부채', '##와', '쿨', '스카프', '##를', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '전파', '##했', '##다', '.', '또', '각', '상점', '##을', '방문', '##해', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '안전', '##하고', '효율', '##적인', '전기', '##기', '##기', '사용법', '##을', '전파', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##하', '##며', '에너지', '절약', '실천', '##을', '당부', '##했', '##다', '.', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '이날', '전기', '##공사', '##협회', '류', '##재', '##선', '회장', '##은', '“', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '올바른', '전기', '사용', '##의', '중요', '##성이', '많이', '홍보', '##됐', '##지만', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '아직', '위험', '##하고', '비', '##효', '##율', '##적으로', '사용', '##되', '##고', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '있', '##다', '”', '며', '시민', '##들', '##의', '관심', '##과', '전기', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '절약', '노력', '##을', '당부', '##했', '##다', '.', '이어', '김성', '##관', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '전기', '##공사', '##공', '##제', '##조합', '이사장', '##은', '“', '안정', '##적인', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '전력', '공급', '##을', '위해', '전력', '##사', '##용', '##량', '##이', '급증', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##하', '##는', '여름철', '효율', '##적인', '냉방', '등', '##으로', '에너지', '##절', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##약', '##을', '실천', '##해', '주시', '##길', '바란다', '”', '고', '말', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##했', '##다', '.', '한국', '##전기', '##공사', '##협회', '##는', '이번', '캠페인', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##이', '안정', '##적인', '전력', '##공', '##급', '##을', '위한', '에너지', '##절', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##약', '##에', '전기', '##건설', '##산업', '##계', '##가', '앞장섰', '##다는', '점', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##에서', '의미', '##가', '있', '##다고', '밝혔', '##다', '.', '[SEP]']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_G3JybH8Mndj",
        "outputId": "a0b0cf61-bc9a-4963-f946-4e390f7459dc"
      },
      "source": [
        "# stride에 적절한 값을 전달 시\n",
        "tokenized_example = tokenizer(example[\"question\"], example[\"context\"], truncation=\"only_second\", stride=4, max_length=24, return_overflowing_tokens=True)\n",
        "\n",
        "for input_ids in tokenized_example[\"input_ids\"]:\n",
        "    print(tokenizer.convert_ids_to_tokens(input_ids))"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '한국', '##전기', '##공사', '##협회', '(', '회장', '류', '##재', '##선', ')', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '류', '##재', '##선', ')', '와', '전기', '##공사', '##공', '##제', '##조합', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##공사', '##공', '##제', '##조합', '(', '이사장', '김성', '##관', ')', '은', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '김성', '##관', ')', '은', '에너지', '##시', '##민', '##연대', '##와', '공동', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##민', '##연대', '##와', '공동', '##으로', '7', '##월', '16', '##일', '서울', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##월', '16', '##일', '서울', '명동', '##에서', '‘', '에너지', '절약', '캠페인', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '‘', '에너지', '절약', '캠페인', '’', '을', '실시', '##했', '##다', '.', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '실시', '##했', '##다', '.', '이번', '캠페인', '##은', '명동', '일대', '상점', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##은', '명동', '일대', '상점', '##과', '시민', '##들', '##에', '##게', '안전', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##들', '##에', '##게', '안전', '##하고', '효율', '##적인', '전기', '##사', '##용', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##적인', '전기', '##사', '##용', '##의', '중요', '##성', '##을', '알리', '##고', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##성', '##을', '알리', '##고', '에너지', '절약', '##의', '필요', '##성', '##을', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##의', '필요', '##성', '##을', '전파', '##하기', '위한', '것', '##으로', ',', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '위한', '것', '##으로', ',', '전력', '##산업', '##계', '##를', '대표', '##하', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##계', '##를', '대표', '##하', '##는', '협회', ',', '조합', ',', '전기', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', ',', '조합', ',', '전기', '##신문', '##사', ',', '전기', '##산업', '##연구원', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', ',', '전기', '##산업', '##연구원', ',', '전기', '##공사', '##공', '##제', '##조합', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##공사', '##공', '##제', '##조합', '##장', '##학', '##회', ',', '엘', '##비', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##회', ',', '엘', '##비', '##라이', '##프', '등', '각', '기관', '임직원', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '등', '각', '기관', '임직원', '##과', '대학생', '자원', '##봉사', '##자', '200', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '자원', '##봉사', '##자', '200', '##여', '##명', '##이', '함께', '##해', '행사', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##이', '함께', '##해', '행사', '##의', '의미', '##를', '더', '##했', '##다', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##를', '더', '##했', '##다', '.', '참가자', '##들', '##은', '행사', '시작', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##들', '##은', '행사', '시작', '##에', '앞서', '전기', '##절', '##약', '##의', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '전기', '##절', '##약', '##의', '중요', '##성', '##을', '담', '##은', '퍼포먼스', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##을', '담', '##은', '퍼포먼스', '##를', '펼친', '후', ',', '명동', '일대', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '후', ',', '명동', '일대', '##를', '행진', '##하', '##며', '시민', '##들이', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##하', '##며', '시민', '##들이', '직접', '실천', '##할', '수', '있', '##는', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##할', '수', '있', '##는', '내용', '##이', '담긴', '부채', '##와', '쿨', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '담긴', '부채', '##와', '쿨', '스카프', '##를', '전파', '##했', '##다', '.', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '전파', '##했', '##다', '.', '또', '각', '상점', '##을', '방문', '##해', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '상점', '##을', '방문', '##해', '안전', '##하고', '효율', '##적인', '전기', '##기', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '효율', '##적인', '전기', '##기', '##기', '사용법', '##을', '전파', '##하', '##며', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##을', '전파', '##하', '##며', '에너지', '절약', '실천', '##을', '당부', '##했', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '실천', '##을', '당부', '##했', '##다', '.', '이날', '전기', '##공사', '##협회', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '이날', '전기', '##공사', '##협회', '류', '##재', '##선', '회장', '##은', '“', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##선', '회장', '##은', '“', '올바른', '전기', '사용', '##의', '중요', '##성이', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '사용', '##의', '중요', '##성이', '많이', '홍보', '##됐', '##지만', '아직', '위험', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##됐', '##지만', '아직', '위험', '##하고', '비', '##효', '##율', '##적으로', '사용', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##효', '##율', '##적으로', '사용', '##되', '##고', '있', '##다', '”', '며', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '있', '##다', '”', '며', '시민', '##들', '##의', '관심', '##과', '전기', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##의', '관심', '##과', '전기', '절약', '노력', '##을', '당부', '##했', '##다', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##을', '당부', '##했', '##다', '.', '이어', '김성', '##관', '전기', '##공사', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '김성', '##관', '전기', '##공사', '##공', '##제', '##조합', '이사장', '##은', '“', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##조합', '이사장', '##은', '“', '안정', '##적인', '전력', '공급', '##을', '위해', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '전력', '공급', '##을', '위해', '전력', '##사', '##용', '##량', '##이', '급증', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##용', '##량', '##이', '급증', '##하', '##는', '여름철', '효율', '##적인', '냉방', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '여름철', '효율', '##적인', '냉방', '등', '##으로', '에너지', '##절', '##약', '##을', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '에너지', '##절', '##약', '##을', '실천', '##해', '주시', '##길', '바란다', '”', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '주시', '##길', '바란다', '”', '고', '말', '##했', '##다', '.', '한국', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##했', '##다', '.', '한국', '##전기', '##공사', '##협회', '##는', '이번', '캠페인', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##협회', '##는', '이번', '캠페인', '##이', '안정', '##적인', '전력', '##공', '##급', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##적인', '전력', '##공', '##급', '##을', '위한', '에너지', '##절', '##약', '##에', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '에너지', '##절', '##약', '##에', '전기', '##건설', '##산업', '##계', '##가', '앞장섰', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##산업', '##계', '##가', '앞장섰', '##다는', '점', '##에서', '의미', '##가', '있', '[SEP]']\n",
            "['[CLS]', '캠페인', '##의', '의의', '##를', '언급', '##한', '단체', '##의', '대표', '##는', '?', '[SEP]', '##에서', '의미', '##가', '있', '##다고', '밝혔', '##다', '.', '[SEP]']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Cw3mvpJiNLaM"
      },
      "source": [
        "동작을 충분히 확인하였으므로, answer가 tokenized context를 구성하는 token들 중 어떤 token들에 해당하는 지 확인하는 코드를 작성해보겠습니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ReG2g4MvPLew",
        "outputId": "77465f35-3af8-4346-e0b0-49935eddc397"
      },
      "source": [
        "# 먼저 우리가 사용할 klue/bert-base가 처리할 수 있는 max_sequence_length를 확인합니다.\n",
        "from transformers import AutoConfig\n",
        "\n",
        "config = AutoConfig.from_pretrained(\"klue/bert-base\")\n",
        "print(config.max_position_embeddings)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "512\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Q9jNzoAD2eve",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "d4db6a50-6a4c-4079-d0a6-03166ea70661"
      },
      "source": [
        "# klue/bert-base가 처리할 수 있는 max_sequence_length보다 긴 길이를 지닌 example을 확인합니다.\n",
        "for i, example in enumerate(train_cs):\n",
        "    if len(tokenizer(example[\"question\"], example[\"context\"])[\"input_ids\"]) > 512:\n",
        "        break\n",
        "example = train_cs[i]\n",
        "# example = train_cs[3]\n",
        "pprint(example)"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Token indices sequence length is longer than the specified maximum sequence length for this model (809 > 512). Running this sequence through the model will result in indexing errors\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'answers': {'answer_start': [198], 'text': ['사비나미술관']},\n",
            " 'context': '미래학자 앨빈 토플러는 2006년 저서 《부의 미래》에서 “3D프린터는 상상하지 못했던 그 무엇이든 만들어낼 수 '\n",
            "            '있다”고 예측했다. 불과 8년이 지난 지금, 그의 말대로 3D(3차원)프린터만 있으면 누구든 자신이 원하는 물건을 '\n",
            "            '디자인해 복잡한 공정을 거치지 않고 제작할 수 있게 됐다. 예술 창작 패러다임에도 커다란 변화의 바람이 불고 '\n",
            "            '있다.서울 안국동 사비나미술관은 이 ‘21세기의 연금술’이 미술에 미치게 될 영향과 미래를 가늠하기 위한 기획전을 '\n",
            "            '연다. 15일부터 7월6일까지 열리는 ‘3D프린팅과 예술: 예술가의 새로운 창작도구’전이다. 예술에서의 3D프린터 '\n",
            "            '활용 가능성은 이미 2000년대 초 일부 학자들에 의해 제기됐다. 그러나 일반에 알려지기 시작한 것은 2007년 '\n",
            "            '월스트리트저널, 타임매거진 등 세계적 언론이 앞다퉈 보도하면서부터다. 19세기 초 사진이 등장해 예술의 패러다임을 '\n",
            "            '획기적으로 바꿔놨듯이 3D프린터는 형태 제작의 한계와 장르의 경계를 무너뜨리고 예술 창조의 영역을 무한대로 확장시킬 '\n",
            "            '것이라는 전망을 쏟아냈다. 특히 상상 속에서만 가능하고 실제로는 구현이 불가능한 형태도 만들어 낼 수 있다는 점은 '\n",
            "            '예술가들에게 희망을 심어줬다. 국내에선 3D프린터의 개발과 보급이 상대적으로 더뎌 예술가들에게는 아직 낯선 게 현실. '\n",
            "            '사비나미술관은 이런 상황을 감안해 대림화학으로부터 보급용 3D프린터와 필라멘트(재료)를 무상으로 지원받는 한편 여러 '\n",
            "            '차례 워크숍을 열어 작가와 토론을 거치며 전시를 꾸렸다.이번 전시에는 3D프린터에 관심을 갖고 있는 신진 및 중견 '\n",
            "            '작가 21명이 참여했다. 1부 ‘상상의 도구, 3D프린터’에서는 이 새로운 도구가 작가의 상상력을 발휘하는 데 어떤 '\n",
            "            '역할을 할 수 있을지 그 가능성을 타진한다. 베른트 할프헤르(독일)는 전쟁과 대립이 지속되고 있는 분쟁 지역의 무기, '\n",
            "            '테러 이미지를 3D프린터로 입체화한 ‘모닝 마운틴즈’를 선보인다. 김승영과 유기태는 3D프린팅의 실패작과 미완성작을 '\n",
            "            '탑처럼 쌓아올려 인간의 불완전성을 상징적으로 제시한다. 댄 마이크셀(미국)은 먼 거리의 미세한 소리를 선택적으로 들을 '\n",
            "            '수 있는 사막여우의 청각기관을 상상적으로 재구성해 3D프린터로 출력했다. 2부 ‘혁신적 복제의 도구, 3D프린터’에는 '\n",
            "            '3D프린터를 통한 재현 기능에 주목한 작품들이 나온다. 3D설계, 3D스캐너를 이용해 인간이 감지할 수 없는 부분을 '\n",
            "            '완벽하게 복제해 보여준다. 권혜원은 동대문역사공원 내 이간수문의 표면을 3D스캐닝과 3D프린팅 방식으로 탁본해낸다. '\n",
            "            '박기진은 종이컵 하나를 재현하는 데 필요한 기계적 연산어를 A4지 17만5000장에 출력해 삶과 기계 사이의 관계를 '\n",
            "            '성찰하게 한다.전시를 기획한 강재현 사비나미술관 전시팀장은 “3D프린터가 가져다 줄 시각예술의 변화를 가늠하고 최첨단 '\n",
            "            '기기를 이용해 거칠지만 개성 있는 형태로 연출된 최초의 전시”라고 의미를 부여했다. (02)736-4371',\n",
            " 'guid': 'klue-mrc-v1_train_07276',\n",
            " 'is_impossible': False,\n",
            " 'news_category': '문화/TV',\n",
            " 'question': '3D프린터가 예술에 끼칠 영향력을 보기 위해 기획전을 연 것은 누구인가?',\n",
            " 'question_type': 1,\n",
            " 'source': 'hankyung',\n",
            " 'title': '미술 창작도 3D프린터 시대'}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "E9Flw4GPPnx1"
      },
      "source": [
        "# 위의 parameter 동작에서 확인한 내용을 적용하여 tokenizer의 __call__ method를 활용합니다.\n",
        "stride=128\n",
        "tokenized_example = tokenizer(example[\"question\"], example[\"context\"], truncation=\"only_second\", return_offsets_mapping=True, return_overflowing_tokens=True, stride=stride)"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2gXiFSMpOPUy",
        "outputId": "b69a222f-3721-45a5-eaab-b31eda2fb6df"
      },
      "source": [
        "# 보통 answer text가 context에서 시작하는 위치정보를 주기 때문에, 아래의 코드로 context에서 어디서부터 어디까지가 answer text에 해당하는 지 확인할 수 있습니다.\n",
        "answers = example[\"answers\"]\n",
        "start_char = answers[\"answer_start\"][0]\n",
        "end_char = start_char + len(answers[\"text\"][0])\n",
        "print(start_char, end_char)\n",
        "print(example[\"context\"][start_char:end_char])"
      ],
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "198 204\n",
            "사비나미술관\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tJKPCTUhRFAC",
        "outputId": "74b2f941-9578-4bd4-89b8-0c4cda6ed17d"
      },
      "source": [
        "# tokenized_example의 sequence_ids method를 활용합니다. 이를 통해서 특정 token이 question에서 왔는 지 context에서 왔는 지 확인할 수 있습니다.\n",
        "# 결과에서 None은 special_token(e.g. [CLS])을 가리킵니다.\n",
        "sequence_ids = tokenized_example.sequence_ids()\n",
        "print(sequence_ids)"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[None, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, None, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, None]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QPzllUmFRcBG",
        "outputId": "9dcc746a-e31f-4e88-a165-e47717c181f9"
      },
      "source": [
        "# tokenized_example의 input_ids 결과에서 context의 시작과 끝 index를 확인할 수 있습니다.\n",
        "# Start token index of the current span in the text.\n",
        "token_start_index = 0\n",
        "while sequence_ids[token_start_index] != 1:\n",
        "    token_start_index += 1\n",
        "# End token index of the current span in the text.\n",
        "token_end_index = len(tokenized_example[\"input_ids\"][0]) - 1\n",
        "while sequence_ids[token_end_index] != 1:\n",
        "    token_end_index -= 1\n",
        "print(token_start_index, token_end_index)"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "26 510\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "-l8dJuoroxXd",
        "outputId": "2b512608-7d26-4059-f14c-721d1c95837d"
      },
      "source": [
        "# offset mapping을 이용하여 answer가 context에 존재하면 token level로 위치를 mapping하고, answer가 context에 존재하지않으면 가장 맨 앞의 [CLS]를 가리키게합니다. \n",
        "# Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n",
        "offsets = tokenized_example[\"offset_mapping\"][0]\n",
        "\n",
        "if (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n",
        "    # Move the token_start_index and token_end_index to the two ends of the answer.\n",
        "    # Note: we could go after the last offset if the answer is the last word (edge case).\n",
        "    while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n",
        "        token_start_index += 1\n",
        "    start_position = token_start_index - 1\n",
        "    while offsets[token_end_index][1] >= end_char:\n",
        "        token_end_index -= 1\n",
        "    end_position = token_end_index + 1\n",
        "    # 아래는 제가 생각한 것\n",
        "    # while token_start_index < len(offsets) and offsets[token_start_index][0] < start_char:\n",
        "    #     token_start_index += 1\n",
        "    # start_position = token_start_index\n",
        "    # while offsets[token_end_index][1] > end_char:\n",
        "    #     token_end_index -= 1\n",
        "    # end_position = token_end_index\n",
        "    print(start_position, end_position)\n",
        "else:\n",
        "    print(\"The answer is not in this feature.\")\n",
        "\n",
        "if start_position == end_position:\n",
        "  print(tokenizer.convert_ids_to_tokens(tokenized_example[\"input_ids\"][0])[start_position:end_position+1])\n",
        "else:\n",
        "  print(tokenizer.convert_ids_to_tokens(tokenized_example[\"input_ids\"][0])[start_position:end_position])"
      ],
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "134 137\n",
            "['사비', '##나', '##미술']\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "kks-teKXTPhU"
      },
      "source": [
        "이 결과를 일반하여 다음의 함수를 작성합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1D1EOG6ZTO9T"
      },
      "source": [
        "from typing import Union, List\n",
        "\n",
        "\n",
        "def prepare_train_features(examples, tokenizer, max_length=512, stride=128):\n",
        "    examples[\"question\"] = [q.lstrip() for q in examples[\"question\"]]\n",
        "    tokenized_examples = tokenizer(\n",
        "        examples[\"question\"],\n",
        "        examples[\"context\"],\n",
        "        truncation=\"only_second\",\n",
        "        max_length=max_length,\n",
        "        stride=stride,\n",
        "        return_overflowing_tokens=True,\n",
        "        return_offsets_mapping=True,\n",
        "    )\n",
        "\n",
        "    # Since one example might give us several features if it has a long context, we need a map from a feature to\n",
        "    # its corresponding example. This key gives us just that.\n",
        "    sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n",
        "    # The offset mappings will give us a map from token to character position in the original context. This will\n",
        "    # help us compute the start_positions and end_positions.\n",
        "    offset_mapping = tokenized_examples.pop(\"offset_mapping\")\n",
        "\n",
        "    # Let's label those examples!\n",
        "    tokenized_examples[\"start_positions\"] = []\n",
        "    tokenized_examples[\"end_positions\"] = []\n",
        "\n",
        "    for i, offsets in enumerate(offset_mapping):\n",
        "        # We will label impossible answers with the index of the CLS token.\n",
        "        input_ids = tokenized_examples[\"input_ids\"][i]\n",
        "        cls_index = input_ids.index(tokenizer.cls_token_id)\n",
        "\n",
        "        # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n",
        "        sequence_ids = tokenized_examples.sequence_ids(i)\n",
        "\n",
        "        # One example can give several spans, this is the index of the example containing this span of text.\n",
        "        sample_index = sample_mapping[i]\n",
        "        answers = examples[\"answers\"][sample_index]\n",
        "\n",
        "        # If no answers are given, set the cls_index as answer.\n",
        "        if len(answers[\"answer_start\"]) == 0:\n",
        "            tokenized_examples[\"start_positions\"].append(cls_index)\n",
        "            tokenized_examples[\"end_positions\"].append(cls_index)\n",
        "        else:\n",
        "            # Start/end character index of the answer in the text.\n",
        "            start_char = answers[\"answer_start\"][0]\n",
        "            end_char = start_char + len(answers[\"text\"][0])\n",
        "\n",
        "            # Start token index of the current span in the text.\n",
        "            token_start_index = 0\n",
        "            while sequence_ids[token_start_index] != 1:\n",
        "                token_start_index += 1\n",
        "\n",
        "            # End token index of the current span in the text.\n",
        "            token_end_index = len(input_ids) - 1\n",
        "            while sequence_ids[token_end_index] != 1:\n",
        "                token_end_index -= 1\n",
        "\n",
        "            # Detect if the answer is out of the span (in which case this feature is labeled with the CLS index).\n",
        "            if not (offsets[token_start_index][0] <= start_char and offsets[token_end_index][1] >= end_char):\n",
        "                tokenized_examples[\"start_positions\"].append(cls_index)\n",
        "                tokenized_examples[\"end_positions\"].append(cls_index)\n",
        "            else:\n",
        "                # Otherwise move the token_start_index and token_end_index to the two ends of the answer.\n",
        "                # Note: we could go after the last offset if the answer is the last word (edge case).\n",
        "                while token_start_index < len(offsets) and offsets[token_start_index][0] <= start_char:\n",
        "                    token_start_index += 1\n",
        "                tokenized_examples[\"start_positions\"].append(token_start_index - 1)\n",
        "                while offsets[token_end_index][1] >= end_char:\n",
        "                    token_end_index -= 1\n",
        "                tokenized_examples[\"end_positions\"].append(token_end_index + 1)\n",
        "\n",
        "    return tokenized_examples"
      ],
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iFspX_7C2z7S",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "a356432a-0477-40e1-e42b-274337dcbcf2"
      },
      "source": [
        "train_ds = train_cs.map(\n",
        "    lambda examples: prepare_train_features(examples, tokenizer, max_length=512, stride=128),\n",
        "    remove_columns=train_cs.column_names,\n",
        "    batched=True\n",
        ")\n",
        "valid_ds = valid_cs.map(\n",
        "    lambda examples: prepare_train_features(examples, tokenizer, max_length=512, stride=128),\n",
        "    remove_columns=valid_cs.column_names,\n",
        "    batched=True\n",
        ")\n",
        "test_ds = test_cs.map(\n",
        "    lambda examples: prepare_train_features(examples, tokenizer, max_length=512, stride=128),\n",
        "    remove_columns=test_cs.column_names,\n",
        "    batched=True\n",
        ")"
      ],
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e/cache-feb750ed450ae791.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e/cache-1bd595d2335dde17.arrow\n",
            "Loading cached processed dataset at /root/.cache/huggingface/datasets/klue/mrc/1.0.0/e0fc3bc3de3eb03be2c92d72fd04a60ecc71903f821619cb28ca0e1e29e4233e/cache-1e1b90a8962ce49f.arrow\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "rccBa-zIsvq_"
      },
      "source": [
        "## Prepare model\n",
        "extractive question answering을 수행하기위해서 `klue/bert-base` load합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OAAkXge5mapO",
        "outputId": "bd2f37f7-8daf-40b6-bdb4-f96e820ecf95"
      },
      "source": [
        "from transformers import AutoModelForQuestionAnswering\n",
        "\n",
        "model = AutoModelForQuestionAnswering.from_pretrained(\"klue/bert-base\")\n",
        "\n",
        "print(model.__class__)"
      ],
      "execution_count": 18,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at klue/bert-base were not used when initializing BertForQuestionAnswering: ['cls.seq_relationship.bias', 'cls.predictions.decoder.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.bias', 'cls.seq_relationship.weight']\n",
            "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at klue/bert-base and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'transformers.models.bert.modeling_bert.BertForQuestionAnswering'>\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zurAlg2TtooX"
      },
      "source": [
        "## Train model\n",
        "`Trainer` class를 이용하여 train합니다.\n",
        "\n",
        "- https://huggingface.co/transformers/custom_datasets.html?highlight=trainer#fine-tuning-with-trainer"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "NGKtGBx4lMdP"
      },
      "source": [
        "from transformers.data.data_collator import DataCollatorWithPadding\n",
        "from datasets import load_metric\n",
        "\n",
        "batchify = DataCollatorWithPadding(\n",
        "    tokenizer=tokenizer,\n",
        "    padding=True\n",
        ")"
      ],
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Brl_Z371mC82",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 572
        },
        "outputId": "0f63b9ca-1806-482a-b8ab-65b4fe92cc82"
      },
      "source": [
        "from transformers import Trainer, TrainingArguments\n",
        "\n",
        "training_args = TrainingArguments(\n",
        "    output_dir='./results',     \n",
        "    evaluation_strategy=\"epoch\",\n",
        "    per_device_train_batch_size=16, \n",
        "    per_device_eval_batch_size=16,\n",
        "    learning_rate=1e-4,\n",
        "    weight_decay=0.01,\n",
        "    adam_beta1=.9,\n",
        "    adam_beta2=.95,\n",
        "    adam_epsilon=1e-8,\n",
        "    max_grad_norm=1.,\n",
        "    num_train_epochs=2,\n",
        "    lr_scheduler_type=\"linear\",\n",
        "    warmup_steps=100,\n",
        "    logging_dir='./logs',\n",
        "    logging_strategy=\"steps\",\n",
        "    logging_first_step=True,\n",
        "    logging_steps=100,\n",
        "    save_strategy=\"epoch\",\n",
        "    seed=42,\n",
        "    dataloader_drop_last=False,\n",
        "    dataloader_num_workers=2\n",
        ")\n",
        "\n",
        "trainer = Trainer(\n",
        "    args=training_args,\n",
        "    data_collator=batchify,\n",
        "    model=model,\n",
        "    train_dataset=train_ds,\n",
        "    eval_dataset=valid_ds,\n",
        ")\n",
        "\n",
        "trainer.train()"
      ],
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running training *****\n",
            "  Num examples = 16894\n",
            "  Num Epochs = 2\n",
            "  Instantaneous batch size per device = 16\n",
            "  Total train batch size (w. parallel, distributed & accumulation) = 16\n",
            "  Gradient Accumulation steps = 1\n",
            "  Total optimization steps = 2112\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='2112' max='2112' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [2112/2112 31:47, Epoch 2/2]\n",
              "    </div>\n",
              "    <table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: left;\">\n",
              "      <th>Epoch</th>\n",
              "      <th>Training Loss</th>\n",
              "      <th>Validation Loss</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <td>1</td>\n",
              "      <td>1.060200</td>\n",
              "      <td>1.038632</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <td>2</td>\n",
              "      <td>0.391900</td>\n",
              "      <td>1.054650</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table><p>"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "***** Running Evaluation *****\n",
            "  Num examples = 1845\n",
            "  Batch size = 16\n",
            "Saving model checkpoint to ./results/checkpoint-1056\n",
            "Configuration saved in ./results/checkpoint-1056/config.json\n",
            "Model weights saved in ./results/checkpoint-1056/pytorch_model.bin\n",
            "***** Running Evaluation *****\n",
            "  Num examples = 1845\n",
            "  Batch size = 16\n",
            "Saving model checkpoint to ./results/checkpoint-2112\n",
            "Configuration saved in ./results/checkpoint-2112/config.json\n",
            "Model weights saved in ./results/checkpoint-2112/pytorch_model.bin\n",
            "\n",
            "\n",
            "Training completed. Do not forget to share your model on huggingface.co/models =)\n",
            "\n",
            "\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "TrainOutput(global_step=2112, training_loss=0.9721131189302965, metrics={'train_runtime': 1908.2646, 'train_samples_per_second': 17.706, 'train_steps_per_second': 1.107, 'total_flos': 8828368195650048.0, 'train_loss': 0.9721131189302965, 'epoch': 2.0})"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "84to5-iqEnZ0"
      },
      "source": [
        "## Evaulate model\n",
        "extractive question answering을 수행하는 model을 평가하기위해서는 다소 복잡한 아래의 과정이 필요합니다.\n",
        "\n",
        "- model이 token 별로 예측한 start, end를 `context`의 token에 mapping하는 과정\n",
        "\n",
        "이 과정을 어떻게 함수화해야하는 지를 예제를 통해서 알아보도록 합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-jenC9DZnjZ5",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9ff3a8d2-b4d9-4d3a-b319-065342c508f4"
      },
      "source": [
        "# mini-batch를 하나 꺼내와서 테스트\n",
        "for batch in trainer.get_test_dataloader(test_ds):\n",
        "    break\n",
        "batch = {k: v.to(trainer.args.device) for k, v in batch.items()}\n",
        "with torch.no_grad():\n",
        "    output = trainer.model(**batch)\n",
        "output.keys()"
      ],
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "odict_keys(['loss', 'start_logits', 'end_logits'])"
            ]
          },
          "metadata": {},
          "execution_count": 21
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "L3v8u28odQ3M"
      },
      "source": [
        "output을 확인해보면 sequence_of_tokens의 token 별로 `start_logits`과 `end_logits`을 가지고 있음을 알 수 있습니다. \n",
        "\n",
        "- https://huggingface.co/transformers/_modules/transformers/models/bert/modeling_bert.html#BertForQuestionAnswering"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "s7-VO3DADsoU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "583b3baa-c455-491d-9755-91f97405b144"
      },
      "source": [
        "output.start_logits.shape, output.end_logits.shape # [16,512,2] -> [16,512,1], [16,512,1] =[16,512], [16,512]"
      ],
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "(torch.Size([16, 512]), torch.Size([16, 512]))"
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "cZLydwstdVZY"
      },
      "source": [
        "각각의 token에 `start_position`인지 `end_position`인지 예측한 score인 `start_logits`, `end_logits`에 대해서, `start_logits`, `end_logits`에 각각 argmax를 하여 mini-batch를 구성하는 각각의 training example의 `start_position`과 `end_position`을 구합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tD2Pz68ydSPe",
        "outputId": "a22e2fc5-79ee-45a6-885c-0d81f18025a9"
      },
      "source": [
        "print(output.start_logits.argmax(dim=-1), output.start_logits.argmax(dim=-1).shape)\n",
        "print(output.end_logits.argmax(dim=-1), output.end_logits.argmax(dim=-1).shape)"
      ],
      "execution_count": 23,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "tensor([352, 301, 296,   0, 297,   0, 217, 439,  67, 397,   0, 124,   0, 325,\n",
            "        197,   0], device='cuda:0') torch.Size([16])\n",
            "tensor([359, 305, 300,   0, 299,   0, 217, 442,  70, 399,   0, 128,   0, 325,\n",
            "        197,   0], device='cuda:0') torch.Size([16])\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "42xFhWwEdgrU"
      },
      "source": [
        "하지만 위의 경우는 아래의 경우에 대해서 대처하지 못할 경우가 있습니다.\n",
        "\n",
        "- `start_position`이 `end_position`보다 뒤에 예측되는 경우\n",
        "- `start_position`과 `end_position`이 question 쪽이 찍히는 경우 \n",
        "\n",
        "따라서 아래와 같이 처리할 필요성이 있습니다.\n",
        "\n",
        "> To classify our answers, we will use the score obtained by adding the start and end logits. We won't try to order all the possible answers and limit ourselves to with a hyper-parameter we call `n_best_size`. We'll pick the best indices in the start and end logits and gather all the answers this predicts. After checking if each one is valid, we will sort them by their score and keep the best one. Here is how we would do this on the first feature in the batch:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xc12dvyZeEwp",
        "outputId": "10652b76-5242-419f-8763-c658ae7a4584"
      },
      "source": [
        "import numpy as np\n",
        "n_best_size = 20\n",
        "# mini-batch 중 하나의 training_example을 가지고와서 예시코드 작성\n",
        "start_logits = output.start_logits[0].cpu().numpy()\n",
        "end_logits = output.end_logits[0].cpu().numpy()\n",
        "\n",
        "# Gather the indices the best start/end logits:\n",
        "start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "valid_answers = []\n",
        "for start_index in start_indexes:\n",
        "    for end_index in end_indexes:\n",
        "        if start_index <= end_index: # We need to refine that test to check the answer is inside the context\n",
        "            valid_answers.append(\n",
        "                {\n",
        "                    \"score\": start_logits[start_index] + end_logits[end_index],\n",
        "                    \"text\": \"\" # We need to find a way to get back the original substring corresponding to the answer in the context\n",
        "                }\n",
        "            )\n",
        "pprint(valid_answers[:5])"
      ],
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[{'score': 12.636349, 'text': ''},\n",
            " {'score': 10.846488, 'text': ''},\n",
            " {'score': 9.155521, 'text': ''},\n",
            " {'score': 5.8207817, 'text': ''},\n",
            " {'score': 5.6747694, 'text': ''}]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5thb9qgOeMAJ"
      },
      "source": [
        "위의 결과에서는 `score`에 맞는 text span을 아직 가져오지 않았습니다. 이에 대응할 수 있도록 아래의 사항을 고려한 작업을 수행해야합니다.\n",
        "\n",
        "- `question`이 아닌 `context`에서 valid span을 가져온다.\n",
        "\n",
        "> To do this, we need to add two things to our validation features:\n",
        " - the ID of the example that generated the feature (since each example can generate several features, as seen before);\n",
        " - the offset mapping that will give us a map from token indices to character positions in the context.\n",
        "\n",
        "이 과정을 처리하는 과정을 예시로 작성해봅니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "IK36nfo9y1vJ",
        "outputId": "bbacdeb1-908f-4d70-8fbc-8bb396b95c27"
      },
      "source": [
        "# Some of the questions have lots of whitespace on the left, which is not useful and will make the\n",
        "# truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n",
        "# left whitespace\n",
        "\n",
        "test_example = test_cs[:1]\n",
        "test_example[\"question\"] = [q.lstrip() for q in test_example[\"question\"]]\n",
        "pprint(test_example)"
      ],
      "execution_count": 25,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "{'answers': [{'answer_start': [666, 666],\n",
            "              'text': ['뉴 740Li 25주년 에디션', '뉴 740Li 25주년']}],\n",
            " 'context': ['BMW 코리아(대표 한상윤)는 창립 25주년을 기념하는 ‘BMW 코리아 25주년 에디션’을 한정 출시한다고 밝혔다. '\n",
            "             '이번 BMW 코리아 25주년 에디션(이하 25주년 에디션)은 BMW 3시리즈와 5시리즈, 7시리즈, 8시리즈 총 '\n",
            "             '4종, 6개 모델로 출시되며, BMW 클래식 모델들로 선보인 바 있는 헤리티지 컬러가 차체에 적용돼 레트로한 느낌과 '\n",
            "             '신구의 조화가 어우러진 차별화된 매력을 자랑한다. 먼저 뉴 320i 및 뉴 320d 25주년 에디션은 트림에 따라 '\n",
            "             '옥스포드 그린(50대 한정) 또는 마카오 블루(50대 한정) 컬러가 적용된다. 럭셔리 라인에 적용되는 옥스포드 '\n",
            "             '그린은 지난 1999년 3세대 3시리즈를 통해 처음 선보인 색상으로 짙은 녹색과 풍부한 펄이 오묘한 조화를 이루는 '\n",
            "             '것이 특징이다. M 스포츠 패키지 트림에 적용되는 마카오 블루는 1988년 2세대 3시리즈를 통해 처음 선보인 바 '\n",
            "             '있으며, 보랏빛 감도는 컬러감이 매력이다. 뉴 520d 25주년 에디션(25대 한정)은 프로즌 브릴리언트 화이트 '\n",
            "             '컬러로 출시된다. BMW가 2011년에 처음 선보인 프로즌 브릴리언트 화이트는 한층 더 환하고 깊은 색감을 '\n",
            "             '자랑하며, 특히 표면을 무광으로 마감해 특별함을 더했다. 뉴 530i 25주년 에디션(25대 한정)은 뉴 3시리즈 '\n",
            "             '25주년 에디션에도 적용된 마카오 블루 컬러가 조합된다. 뉴 740Li 25주년 에디션(7대 한정)에는 말라카이트 '\n",
            "             '그린 다크 색상이 적용된다. 잔잔하면서도 오묘한 깊은 녹색을 발산하는 말라카이트 그린 다크는 장식재로 활용되는 광물 '\n",
            "             '말라카이트에서 유래됐다. 뉴 840i xDrive 그란쿠페 25주년 에디션(8대 한정)은 인도양의 맑고 투명한 '\n",
            "             '에메랄드 빛을 연상케 하는 몰디브 블루 컬러로 출시된다. 특히 몰디브 블루는 지난 1993년 1세대 8시리즈에 '\n",
            "             '처음으로 적용되었던 만큼 이를 오마주하는 의미를 담고 있다.'],\n",
            " 'guid': ['klue-mrc-v1_dev_01891'],\n",
            " 'is_impossible': [False],\n",
            " 'news_category': ['자동차'],\n",
            " 'question': ['말라카이트에서 나온 색깔을 사용한 에디션은?'],\n",
            " 'question_type': [2],\n",
            " 'source': ['acrofan'],\n",
            " 'title': ['BMW 코리아, 창립 25주년 기념 ‘BMW 코리아 25주년 에디션’ 한정 출시']}\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Fby-n62wzBvJ"
      },
      "source": [
        "# Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n",
        "# in one example possible giving several features when a context is long, each of those features having a\n",
        "# context that overlaps a bit the context of the previous feature.\n",
        "tokenized_test_example = tokenizer(\n",
        "    test_example[\"question\"],\n",
        "    test_example[\"context\"],\n",
        "    truncation=\"only_second\",\n",
        "    max_length=512,\n",
        "    stride=128,\n",
        "    return_overflowing_tokens=True,\n",
        "    return_offsets_mapping=True,\n",
        "    padding=\"max_length\",\n",
        ")"
      ],
      "execution_count": 26,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EdwMOV4qzafH"
      },
      "source": [
        "# Since one example might give us several features if it has a long context, we need a map from a feature to\n",
        "# its corresponding example. This key gives us just that.\n",
        "sample_mapping = tokenized_test_example.pop(\"overflow_to_sample_mapping\")"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NKYHurnazb7k",
        "outputId": "d0a2abf5-e6da-4ef4-8381-64c70ffa357b"
      },
      "source": [
        "# We keep the example_id that gave us this feature and we will store the offset mappings.\n",
        "tokenized_test_example[\"example_id\"] = []\n",
        "\n",
        "for i in range(len(tokenized_test_example[\"input_ids\"])):\n",
        "    # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n",
        "    sequence_ids = tokenized_test_example.sequence_ids(i)\n",
        "    context_index = 1\n",
        "    # One example can give several spans, this is the index of the example containing this span of text.\n",
        "    sample_index = sample_mapping[i]\n",
        "    tokenized_test_example[\"example_id\"].append(test_example[\"guid\"][sample_index])\n",
        "\n",
        "    # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n",
        "    # position is part of the context or not.\n",
        "    tokenized_test_example[\"offset_mapping\"][i] = [\n",
        "        (o if sequence_ids[k] == context_index else None)\n",
        "        for k, o in enumerate(tokenized_test_example[\"offset_mapping\"][i])\n",
        "    ]\n",
        "\n",
        "print(tokenized_test_example[\"example_id\"])\n",
        "print(tokenized_test_example[\"offset_mapping\"])"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['klue-mrc-v1_dev_01891']\n",
            "[[None, None, None, None, None, None, None, None, None, None, None, None, None, None, (0, 3), (4, 7), (7, 8), (8, 10), (11, 13), (13, 14), (14, 15), (15, 16), (17, 19), (20, 22), (22, 23), (23, 24), (24, 25), (26, 28), (28, 29), (29, 30), (31, 32), (32, 35), (36, 39), (40, 42), (42, 43), (43, 44), (45, 48), (48, 49), (49, 50), (51, 53), (54, 56), (56, 58), (58, 59), (60, 62), (62, 63), (63, 64), (65, 67), (68, 71), (72, 75), (76, 78), (78, 79), (79, 80), (81, 84), (84, 85), (85, 87), (88, 90), (90, 91), (91, 92), (93, 96), (96, 97), (97, 98), (99, 102), (103, 104), (104, 105), (105, 106), (106, 107), (107, 108), (109, 110), (110, 111), (111, 112), (112, 113), (113, 114), (115, 116), (116, 117), (117, 118), (118, 119), (119, 120), (121, 122), (122, 123), (123, 124), (124, 125), (126, 127), (128, 129), (129, 130), (130, 131), (132, 133), (133, 134), (135, 137), (137, 138), (139, 141), (141, 142), (142, 143), (143, 144), (145, 148), (149, 152), (153, 155), (155, 156), (156, 157), (158, 161), (162, 163), (164, 165), (165, 166), (167, 168), (168, 170), (170, 171), (172, 174), (174, 175), (176, 178), (178, 179), (180, 182), (182, 183), (184, 186), (186, 187), (187, 188), (189, 191), (191, 192), (193, 195), (195, 196), (197, 199), (199, 200), (201, 205), (206, 208), (208, 209), (209, 210), (211, 213), (213, 214), (215, 217), (217, 219), (219, 220), (221, 223), (224, 225), (226, 229), (229, 230), (231, 232), (233, 234), (235, 238), (238, 239), (240, 242), (242, 243), (243, 244), (245, 248), (248, 249), (250, 252), (252, 253), (254, 256), (257, 259), (259, 261), (262, 264), (264, 265), (265, 267), (267, 268), (269, 271), (271, 272), (273, 275), (276, 279), (280, 282), (282, 283), (283, 285), (285, 286), (287, 289), (289, 290), (291, 293), (293, 294), (295, 297), (297, 298), (298, 299), (299, 300), (301, 304), (305, 307), (307, 308), (309, 311), (311, 312), (312, 313), (314, 316), (316, 318), (319, 321), (321, 322), (323, 325), (326, 330), (330, 331), (332, 333), (333, 335), (336, 337), (337, 338), (338, 339), (339, 340), (340, 341), (342, 344), (345, 347), (348, 351), (352, 354), (354, 356), (357, 358), (358, 359), (360, 362), (362, 363), (364, 366), (366, 367), (368, 369), (369, 370), (371, 373), (373, 374), (375, 377), (377, 378), (379, 381), (381, 382), (383, 384), (384, 385), (386, 388), (388, 390), (390, 391), (392, 393), (394, 397), (398, 401), (402, 404), (404, 405), (406, 408), (408, 409), (409, 410), (411, 414), (415, 417), (417, 418), (419, 423), (423, 424), (425, 426), (426, 428), (429, 430), (430, 431), (431, 432), (432, 433), (433, 434), (435, 437), (438, 440), (441, 444), (445, 446), (447, 448), (448, 450), (450, 451), (452, 453), (453, 454), (454, 455), (456, 458), (458, 459), (460, 462), (462, 463), (463, 464), (465, 467), (467, 469), (469, 470), (471, 472), (473, 476), (476, 477), (478, 480), (480, 481), (481, 482), (483, 486), (486, 487), (487, 489), (489, 490), (491, 493), (493, 494), (494, 495), (496, 498), (498, 499), (500, 501), (501, 502), (502, 503), (503, 504), (504, 505), (506, 509), (510, 512), (512, 513), (514, 516), (516, 517), (517, 518), (518, 519), (520, 523), (523, 524), (525, 529), (529, 530), (530, 531), (532, 534), (535, 538), (539, 541), (541, 542), (543, 544), (544, 545), (545, 546), (546, 547), (547, 548), (549, 552), (552, 553), (554, 556), (557, 558), (559, 561), (561, 562), (563, 564), (564, 565), (566, 568), (568, 569), (570, 572), (572, 573), (573, 574), (574, 575), (576, 578), (579, 581), (581, 582), (583, 584), (584, 585), (585, 587), (588, 590), (590, 591), (592, 594), (594, 595), (595, 596), (597, 598), (598, 599), (599, 600), (600, 601), (602, 603), (604, 607), (607, 608), (609, 611), (611, 612), (612, 613), (614, 617), (617, 618), (618, 620), (620, 621), (622, 624), (624, 625), (625, 626), (627, 628), (629, 630), (630, 631), (631, 632), (632, 633), (634, 636), (636, 637), (637, 638), (639, 642), (642, 644), (645, 647), (647, 648), (649, 652), (653, 655), (656, 658), (658, 659), (660, 662), (662, 663), (663, 664), (664, 665), (666, 667), (668, 671), (671, 672), (672, 673), (674, 676), (676, 677), (677, 678), (679, 682), (682, 683), (683, 684), (684, 685), (686, 688), (688, 689), (689, 691), (692, 694), (694, 696), (696, 697), (698, 700), (701, 703), (704, 706), (706, 707), (708, 710), (710, 711), (711, 712), (712, 713), (714, 716), (716, 718), (718, 719), (719, 720), (721, 723), (723, 724), (725, 726), (726, 727), (728, 730), (730, 731), (732, 734), (734, 735), (735, 736), (737, 739), (739, 741), (741, 742), (743, 745), (746, 748), (748, 749), (750, 752), (752, 753), (753, 754), (755, 757), (757, 758), (758, 759), (760, 762), (763, 765), (765, 767), (767, 768), (768, 770), (771, 773), (773, 774), (774, 775), (775, 776), (777, 778), (779, 781), (781, 782), (782, 783), (784, 785), (785, 786), (786, 789), (789, 790), (791, 793), (793, 794), (794, 795), (796, 798), (798, 799), (799, 800), (801, 804), (804, 805), (805, 806), (806, 807), (808, 810), (810, 811), (811, 812), (813, 815), (815, 816), (816, 817), (818, 819), (819, 820), (821, 823), (823, 824), (825, 827), (827, 828), (828, 829), (830, 831), (831, 832), (833, 835), (835, 836), (837, 838), (838, 839), (840, 841), (841, 842), (842, 843), (844, 846), (847, 849), (849, 850), (851, 853), (853, 854), (854, 855), (855, 856), (857, 859), (860, 861), (861, 862), (862, 863), (864, 866), (866, 867), (868, 870), (871, 875), (875, 876), (877, 878), (878, 880), (881, 882), (882, 883), (883, 884), (884, 885), (885, 886), (887, 889), (889, 891), (892, 894), (894, 895), (895, 896), (896, 897), (898, 900), (901, 903), (904, 906), (906, 907), (907, 908), (908, 909), (910, 912), (912, 913), (914, 915), (915, 916), (917, 918), (918, 919), (919, 920), None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None, None]]\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lcFaVz39eIDO"
      },
      "source": [
        "def prepare_validation_features(examples, tokenizer, max_length=512, stride=128):\n",
        "    # Some of the questions have lots of whitespace on the left, which is not useful and will make the\n",
        "    # truncation of the context fail (the tokenized question will take a lots of space). So we remove that\n",
        "    # left whitespace\n",
        "    examples[\"question\"] = [q.lstrip() for q in examples[\"question\"]]\n",
        "\n",
        "    # Tokenize our examples with truncation and maybe padding, but keep the overflows using a stride. This results\n",
        "    # in one example possible giving several features when a context is long, each of those features having a\n",
        "    # context that overlaps a bit the context of the previous feature.\n",
        "    tokenized_examples = tokenizer(\n",
        "        examples[\"question\"],\n",
        "        examples[\"context\"],\n",
        "        truncation=\"only_second\",\n",
        "        max_length=max_length,\n",
        "        stride=stride,\n",
        "        return_overflowing_tokens=True,\n",
        "        return_offsets_mapping=True,\n",
        "        padding=\"max_length\",\n",
        "    )\n",
        "\n",
        "    # Since one example might give us several features if it has a long context, we need a map from a feature to\n",
        "    # its corresponding example. This key gives us just that.\n",
        "    sample_mapping = tokenized_examples.pop(\"overflow_to_sample_mapping\")\n",
        "\n",
        "    # We keep the example_id that gave us this feature and we will store the offset mappings.\n",
        "    tokenized_examples[\"example_id\"] = []\n",
        "\n",
        "    for i in range(len(tokenized_examples[\"input_ids\"])):\n",
        "        # Grab the sequence corresponding to that example (to know what is the context and what is the question).\n",
        "        sequence_ids = tokenized_examples.sequence_ids(i)\n",
        "        context_index = 1\n",
        "        # One example can give several spans, this is the index of the example containing this span of text.\n",
        "        sample_index = sample_mapping[i]\n",
        "        tokenized_examples[\"example_id\"].append(examples[\"guid\"][sample_index])\n",
        "\n",
        "        # Set to None the offset_mapping that are not part of the context so it's easy to determine if a token\n",
        "        # position is part of the context or not.\n",
        "        tokenized_examples[\"offset_mapping\"][i] = [\n",
        "            (o if sequence_ids[k] == context_index else None)\n",
        "            for k, o in enumerate(tokenized_examples[\"offset_mapping\"][i])\n",
        "        ]\n",
        "\n",
        "    return tokenized_examples"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "aoXINlA1r_RM"
      },
      "source": [
        "And like before, we can apply that function to our validation set easily:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v9yNtu9QhGhl",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 49,
          "referenced_widgets": [
            "08baf1a74a2c4cebbb5dd83030e25c69",
            "a95ecb9980f14ac691d4e30fe2036d9b",
            "86d8497adcb640c8ba7dc961fbf806ec",
            "0df74cb5fe494f2faf8c7c57c7220ab2",
            "43524da14fcf42f9a92740eda8e3078b",
            "1879c3bb390943849ef9ca934ca20a99",
            "7240ca258f204945952347c32c37c749",
            "e5f00a84e3f84abfbb81253c9322bb9f",
            "94f3779636ac4301b83614cfe0cbb673",
            "d7ca52c582584303aa8f455d98589181",
            "008e655d6f61432f86cb1d5502fb74a1"
          ]
        },
        "outputId": "a3ca0fe7-18ef-4a3b-82fc-6c3dd25c11da"
      },
      "source": [
        "test_features = test_cs.map(\n",
        "    lambda examples: prepare_validation_features(examples, tokenizer, max_length=512, stride=128),\n",
        "    batched=True,\n",
        "    remove_columns=test_cs.column_names\n",
        ")"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "08baf1a74a2c4cebbb5dd83030e25c69",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/5 [00:00<?, ?ba/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "jvCvX8V8upe_"
      },
      "source": [
        "새롭게 정의한 `prepare_validation_features` function을 이용해서 transform된 example에 대해서 prediction을 수행합니다."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 106
        },
        "id": "KrzDvoL6uOJ0",
        "outputId": "bb89af23-c96b-4931-947b-72964f611aac"
      },
      "source": [
        "test_predictions = trainer.predict(test_features)"
      ],
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "The following columns in the test set  don't have a corresponding argument in `BertForQuestionAnswering.forward` and have been ignored: example_id, offset_mapping.\n",
            "***** Running Prediction *****\n",
            "  Num examples = 6268\n",
            "  Batch size = 16\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "\n",
              "    <div>\n",
              "      \n",
              "      <progress value='392' max='392' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
              "      [392/392 01:59]\n",
              "    </div>\n",
              "    "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "5BtoIXvt84ms"
      },
      "source": [
        "~*The* `Trainer` *hides* the columns that are not used by the model (here `example_id` and `offset_mapping` which we will need for our post-processing), so we set them back:~"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M0SAvHg6usKV"
      },
      "source": [
        "# test_features.set_format(type=test_features.format[\"type\"], columns=list(test_features.features.keys()))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "EVqBUM1w8_tZ"
      },
      "source": [
        "We can now refine the test we had before: since we set `None` in the offset mappings when it corresponds to a part of the question, it's easy to check if an answer is fully inside the context. We also eliminate very long answers from our considerations (with an hyper-parameter we can tune)"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SDLB5qET88xd"
      },
      "source": [
        "max_answer_length = 30"
      ],
      "execution_count": 32,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6Kik4wvx8_Io",
        "outputId": "4127c2f5-52db-48e6-f5c4-71f1070790c8"
      },
      "source": [
        "start_logits = output.start_logits[0].cpu().numpy()\n",
        "end_logits = output.end_logits[0].cpu().numpy()\n",
        "offset_mapping = test_features[0][\"offset_mapping\"]\n",
        "# The first feature comes from the first example. For the more general case, we will need to be match the example_id to\n",
        "# an example index\n",
        "context = test_cs[0][\"context\"]\n",
        "\n",
        "# Gather the indices the best start/end logits:\n",
        "start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "valid_answers = []\n",
        "for start_index in start_indexes:\n",
        "    for end_index in end_indexes:\n",
        "        # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n",
        "        # to part of the input_ids that are not in the context.\n",
        "        if (\n",
        "            start_index >= len(offset_mapping)\n",
        "            or end_index >= len(offset_mapping)\n",
        "            or offset_mapping[start_index] is None\n",
        "            or offset_mapping[end_index] is None\n",
        "        ):\n",
        "            continue\n",
        "        # Don't consider answers with a length that is either < 0 or > max_answer_length.\n",
        "        if end_index < start_index or end_index - start_index + 1 > max_answer_length:\n",
        "            continue\n",
        "        if start_index <= end_index: # We need to refine that test to check the answer is inside the context\n",
        "            start_char = offset_mapping[start_index][0]\n",
        "            end_char = offset_mapping[end_index][1]\n",
        "            valid_answers.append(\n",
        "                {\n",
        "                    \"score\": start_logits[start_index] + end_logits[end_index],\n",
        "                    \"text\": context[start_char: end_char]\n",
        "                }\n",
        "            )\n",
        "\n",
        "valid_answers = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[:n_best_size]\n",
        "valid_answers"
      ],
      "execution_count": 33,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'score': 12.636349, 'text': '뉴 740Li 25주년 에디션'},\n",
              " {'score': 10.846488, 'text': '뉴 740Li 25주년 에디션(7대 한정)'},\n",
              " {'score': 9.155521, 'text': '뉴 740Li'},\n",
              " {'score': 8.717138, 'text': '25주년 에디션'},\n",
              " {'score': 6.9272776, 'text': '25주년 에디션(7대 한정)'},\n",
              " {'score': 6.6313076, 'text': '(7대 한정)'},\n",
              " {'score': 5.6747694, 'text': '뉴 740Li 25주년'},\n",
              " {'score': 4.7079053, 'text': '뉴 740Li 25주년 에디션(7대 한정)에는 말라카이트 그린 다크'},\n",
              " {'score': 4.456806, 'text': '740Li 25주년 에디션'},\n",
              " {'score': 4.151575, 'text': '뉴 740Li 25'},\n",
              " {'score': 3.9307623,\n",
              "  'text': '뉴 3시리즈 25주년 에디션에도 적용된 마카오 블루 컬러가 조합된다. 뉴 740Li 25주년 에디션'},\n",
              " {'score': 2.666946, 'text': '740Li 25주년 에디션(7대 한정)'},\n",
              " {'score': 1.7555588, 'text': '25주년'},\n",
              " {'score': 0.97597945, 'text': '740Li'},\n",
              " {'score': 0.953266, 'text': '뉴 530i 25주년 에디션'},\n",
              " {'score': 0.788695, 'text': '25주년 에디션(7대 한정)에는 말라카이트 그린 다크'},\n",
              " {'score': 0.492725, 'text': '(7대 한정)에는 말라카이트 그린 다크'},\n",
              " {'score': 0.44993544,\n",
              "  'text': '뉴 3시리즈 25주년 에디션에도 적용된 마카오 블루 컬러가 조합된다. 뉴 740Li'},\n",
              " {'score': 0.2323649, 'text': '25'},\n",
              " {'score': -0.34390414, 'text': '‘BMW 코리아 25주년 에디션’'}]"
            ]
          },
          "metadata": {},
          "execution_count": 33
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "RiqfL7YS9QUU"
      },
      "source": [
        "We can compare to the actual ground-truth answer:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "iFUP6hOE9K92",
        "outputId": "f480609a-6f15-4817-e356-614cd78c9f20"
      },
      "source": [
        "test_cs[0][\"answers\"]"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'answer_start': [666, 666], 'text': ['뉴 740Li 25주년 에디션', '뉴 740Li 25주년']}"
            ]
          },
          "metadata": {},
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JZyugcdR9Vkw"
      },
      "source": [
        "Our model picked the right as the most likely answer!\n",
        "\n",
        "As we mentioned in the code above, this was easy on the first feature because we knew it comes from the first example. For the other features, we will need a map between examples and their corresponding features. Also, since one example can give several features, we will need to gather together all the answers in all the features generated by a given example, then pick the best one. The following code builds a map from example index to its corresponding features indices:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "K49knhyM9R2A"
      },
      "source": [
        "import collections\n",
        "\n",
        "examples = test_cs\n",
        "features = test_features\n",
        "\n",
        "example_id_to_index = {k: i for i, k in enumerate(examples[\"guid\"])}\n",
        "features_per_example = collections.defaultdict(list)\n",
        "for i, feature in enumerate(features):\n",
        "    features_per_example[example_id_to_index[feature[\"example_id\"]]].append(i)"
      ],
      "execution_count": 35,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mEwbSPEf9nPg"
      },
      "source": [
        "We're almost ready for our post-processing function. The last bit to deal with is the impossible answer (when `squad_v2 = True`). The code above only keeps answers that are inside the context, we need to also grab the score for the impossible answer (which has start and end indices corresponding to the index of the CLS token). When one example gives several features, we have to predict the impossible answer when all the features give a high score to the impossible answer (since one feature could predict the impossible answer just because the answer isn't in the part of the context it has access too), which is why the score of the impossible answer for one example is the *minimum* of the scores for the impossible answer in each feature generated by the example.\n",
        "\n",
        "We then predict the impossible answer when that score is greater than the score of the best non-impossible answer. All combined together, this gives us this post-processing function:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FRyYno9A9geX"
      },
      "source": [
        "from tqdm.auto import tqdm\n",
        "\n",
        "def postprocess_qa_predictions(examples, features, raw_predictions, n_best_size = 20, max_answer_length = 30):\n",
        "    all_start_logits, all_end_logits = raw_predictions\n",
        "    # Build a map example to its corresponding features.\n",
        "    example_id_to_index = {k: i for i, k in enumerate(examples[\"guid\"])}\n",
        "    features_per_example = collections.defaultdict(list)\n",
        "    for i, feature in enumerate(features):\n",
        "        features_per_example[example_id_to_index[feature[\"example_id\"]]].append(i)\n",
        "\n",
        "    # The dictionaries we have to fill.\n",
        "    predictions = collections.OrderedDict()\n",
        "\n",
        "    # Logging.\n",
        "    print(f\"Post-processing {len(examples)} example predictions split into {len(features)} features.\")\n",
        "\n",
        "    # Let's loop over all the examples!\n",
        "    for example_index, example in enumerate(tqdm(examples)):\n",
        "        # Those are the indices of the features associated to the current example.\n",
        "        feature_indices = features_per_example[example_index]\n",
        "\n",
        "        min_null_score = None # Only used if squad_v2 is True.\n",
        "        valid_answers = []\n",
        "        \n",
        "        context = example[\"context\"]\n",
        "        # Looping through all the features associated to the current example.\n",
        "        for feature_index in feature_indices:\n",
        "            # We grab the predictions of the model for this feature.\n",
        "            start_logits = all_start_logits[feature_index]\n",
        "            end_logits = all_end_logits[feature_index]\n",
        "            # This is what will allow us to map some the positions in our logits to span of texts in the original\n",
        "            # context.\n",
        "            offset_mapping = features[feature_index][\"offset_mapping\"]\n",
        "\n",
        "            # Update minimum null prediction.\n",
        "            cls_index = features[feature_index][\"input_ids\"].index(tokenizer.cls_token_id)\n",
        "            feature_null_score = start_logits[cls_index] + end_logits[cls_index]\n",
        "            if min_null_score is None or min_null_score < feature_null_score:\n",
        "                min_null_score = feature_null_score\n",
        "\n",
        "            # Go through all possibilities for the `n_best_size` greater start and end logits.\n",
        "            start_indexes = np.argsort(start_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "            end_indexes = np.argsort(end_logits)[-1 : -n_best_size - 1 : -1].tolist()\n",
        "            for start_index in start_indexes:\n",
        "                for end_index in end_indexes:\n",
        "                    # Don't consider out-of-scope answers, either because the indices are out of bounds or correspond\n",
        "                    # to part of the input_ids that are not in the context.\n",
        "                    if (\n",
        "                        start_index >= len(offset_mapping)\n",
        "                        or end_index >= len(offset_mapping)\n",
        "                        or offset_mapping[start_index] is None\n",
        "                        or offset_mapping[end_index] is None\n",
        "                    ):\n",
        "                        continue\n",
        "                    # Don't consider answers with a length that is either < 0 or > max_answer_length.\n",
        "                    if end_index < start_index or end_index - start_index + 1 > max_answer_length:\n",
        "                        continue\n",
        "\n",
        "                    start_char = offset_mapping[start_index][0]\n",
        "                    end_char = offset_mapping[end_index][1]\n",
        "                    valid_answers.append(\n",
        "                        {\n",
        "                            \"score\": start_logits[start_index] + end_logits[end_index],\n",
        "                            \"text\": context[start_char: end_char]\n",
        "                        }\n",
        "                    )\n",
        "        \n",
        "        if len(valid_answers) > 0:\n",
        "            best_answer = sorted(valid_answers, key=lambda x: x[\"score\"], reverse=True)[0]\n",
        "        else:\n",
        "            # In the very rare edge case we have not a single non-null prediction, we create a fake prediction to avoid\n",
        "            # failure.\n",
        "            best_answer = {\"text\": \"\", \"score\": 0.0}\n",
        "        \n",
        "        # Let's pick our final answer: the best one or the null answer (only for squad_v2)\n",
        "        # if not squad_v2:\n",
        "        #     predictions[example[\"id\"]] = best_answer[\"text\"]\n",
        "        # else:\n",
        "        #     answer = best_answer[\"text\"] if best_answer[\"score\"] > min_null_score else \"\"\n",
        "        #     predictions[example[\"id\"]] = answer\n",
        "        predictions[example[\"guid\"]] = best_answer[\"text\"]\n",
        "\n",
        "    return predictions"
      ],
      "execution_count": 36,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fYF9KOU2-Gsf"
      },
      "source": [
        "And we can apply our post-processing function to our raw predictions:"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "1436af3e6fe347d1b067ef729e7dedc7",
            "32ace2b72e594204994fae0d3e465bc7",
            "c2d1731c3ce54d30ae9e88d6643688d3",
            "0fcfa7dd647e4301a635429de76af648",
            "53d14a74367440269353e7e9778ee85a",
            "56c75e6326ec4601906c687dbc96c341",
            "2708e34134e14c0ca3d87c7b2c056007",
            "6f5e85f6903e48d994eaafb9263abb0c",
            "3a0f87b9409f45c89af5fb6265244f09",
            "1661341a9dac418ab998b1dffccab0b3",
            "2bcda5cf72404e0cb28d790c639c4e29"
          ]
        },
        "id": "qCyeDN7D-DeF",
        "outputId": "fe95e3bf-eec5-4b41-b9c9-c6d85bdbb7c4"
      },
      "source": [
        "final_predictions = postprocess_qa_predictions(test_cs, test_features, test_predictions.predictions)"
      ],
      "execution_count": 37,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Post-processing 4008 example predictions split into 6268 features.\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1436af3e6fe347d1b067ef729e7dedc7",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "  0%|          | 0/4008 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "xVivLeJv-Yr_"
      },
      "source": [
        "Then we can load the metric from the datasets library."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81,
          "referenced_widgets": [
            "4f4b214805db4b1bba7855293477ba96",
            "c2d5527a92c24c68a3b3bb1ee5f4d5bb",
            "0d8620a2cd1f4543981dc5e05bd48c80",
            "16bdc720b44043b58cddccea79eb5a19",
            "9e5286cef5af44229ef52fd637712b3e",
            "9e4a8203b5f64be3bc98d960e94efb0f",
            "d5496e20c1e94c80910b74f2e7e5a00b",
            "a8d91772e23842099a27c0065ee6b5f4",
            "a549b79c10ec49f4b01cc4ce81fcea92",
            "51140dd903494f16a3cb1f22a0ce9092",
            "8f8c3bbe1ca243939b216096e1845a54",
            "d8a558407dad46969d40a33088eb3121",
            "55c63e2653144dfa9eb73116ca70b5f9",
            "2b31942058ac4a6bb20a436828407164",
            "7bda3b5b23e141ddb416e6521ce7de51",
            "77f6db4122174db990c368ce2ca964cc",
            "bb255d7240724fe88fabd18c1edffbb7",
            "b7a1d4b8848e4c15b6cf041e7d4370dc",
            "1c863110097e45cb9512876c0fed9598",
            "5b6562ba44e949a2904bda8c17ea5630",
            "9bb5ff8b97c445d7ae7573ce234bfece",
            "7420737a61ea452684d35a4c9268152d"
          ]
        },
        "id": "_qp1qX66-PpD",
        "outputId": "663cf5de-4cba-4c5d-fe6a-adfb2c46c269"
      },
      "source": [
        "# metric = load_metric(\"squad_v2\" if squad_v2 else \"squad\")\n",
        "metric = load_metric(\"squad\")"
      ],
      "execution_count": 38,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "4f4b214805db4b1bba7855293477ba96",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.73k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d8a558407dad46969d40a33088eb3121",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.12k [00:00<?, ?B/s]"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Ac-vr5g-ikG"
      },
      "source": [
        "Then we can call compute on it. We just need to format predictions and labels a bit as it expects a list of dictionaries and not one big dictionary. In the case of squad_v2, we also have to set a `no_answer_probability` argument (which we set to 0.0 here as we have already set the answer to empty if we picked it)."
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "jztRO3Gd-d5Y",
        "outputId": "fe66ba39-d532-4d4d-fab1-a5631f9d9e74"
      },
      "source": [
        "# if squad_v2:\n",
        "#     formatted_predictions = [{\"id\": k, \"prediction_text\": v, \"no_answer_probability\": 0.0} for k, v in final_predictions.items()]\n",
        "# else:\n",
        "#     formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in final_predictions.items()]\n",
        "formatted_predictions = [{\"id\": k, \"prediction_text\": v} for k, v in final_predictions.items()]\n",
        "references = [{\"id\": ex[\"guid\"], \"answers\": ex[\"answers\"]} for ex in test_cs]\n",
        "metric.compute(predictions=formatted_predictions, references=references, )"
      ],
      "execution_count": 39,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'exact_match': 69.76047904191617, 'f1': 73.7957510336649}"
            ]
          },
          "metadata": {},
          "execution_count": 39
        }
      ]
    }
  ]
}