Document_Search_Own_Data_7th_Sept_23.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "gpuType": "T4",
      "authorship_tag": "ABX9TyPICvDxePiO9h9s9Cm6DCZ0",
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a04e2e89bfb1494d82cfba09c596c1bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b3db777fee414a78b15c8a3c1dc7767a",
              "IPY_MODEL_6cd0fbb46203482189c0b418ca94b325",
              "IPY_MODEL_03e1fb810bbd4ca4b1956a8ec4ccdad1"
            ],
            "layout": "IPY_MODEL_13217588ae194d438d486de317f6eecb"
          }
        },
        "b3db777fee414a78b15c8a3c1dc7767a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_42edf9a594dc4a6884e7d1171b6c480c",
            "placeholder": "​",
            "style": "IPY_MODEL_60407a2a68e640429525423e50ea9c31",
            "value": "Downloading (…)lve/main/config.json: 100%"
          }
        },
        "6cd0fbb46203482189c0b418ca94b325": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3952e294794d461db461c4924191aef5",
            "max": 508,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6b22b383941644cc8560be08e6072bac",
            "value": 508
          }
        },
        "03e1fb810bbd4ca4b1956a8ec4ccdad1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c92bc3d4b894c8ca597253b7c4cde13",
            "placeholder": "​",
            "style": "IPY_MODEL_9efd3aac223146559b27191a80c3bc1f",
            "value": " 508/508 [00:00&lt;00:00, 22.0kB/s]"
          }
        },
        "13217588ae194d438d486de317f6eecb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "42edf9a594dc4a6884e7d1171b6c480c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "60407a2a68e640429525423e50ea9c31": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3952e294794d461db461c4924191aef5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6b22b383941644cc8560be08e6072bac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0c92bc3d4b894c8ca597253b7c4cde13": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "9efd3aac223146559b27191a80c3bc1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "71bad721f024403b87a8379de152571a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_927efe7a40314379a659d807b1be44bb",
              "IPY_MODEL_5d41705b6a694a0baa31bba07c9f4a10",
              "IPY_MODEL_372fc7aa0a5e4d53956677f807f2f497"
            ],
            "layout": "IPY_MODEL_aaf7b5905af04b6e97acf1b4dbe14aeb"
          }
        },
        "927efe7a40314379a659d807b1be44bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a55f64e1336b4558bdc5cfb3c45b65cd",
            "placeholder": "​",
            "style": "IPY_MODEL_6b913c98d7774f26b45c864e679cade2",
            "value": "Downloading model.safetensors: 100%"
          }
        },
        "5d41705b6a694a0baa31bba07c9f4a10": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e55e2dfa2f0b4b5ab53eea8e6f4a023f",
            "max": 433270764,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b909e582846e43fb8380e9916fbc58f0",
            "value": 433270764
          }
        },
        "372fc7aa0a5e4d53956677f807f2f497": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e1d0477866bc42c783c866c3f4089623",
            "placeholder": "​",
            "style": "IPY_MODEL_0aa40681679b4b83a4ebebb24b674575",
            "value": " 433M/433M [00:02&lt;00:00, 160MB/s]"
          }
        },
        "aaf7b5905af04b6e97acf1b4dbe14aeb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a55f64e1336b4558bdc5cfb3c45b65cd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6b913c98d7774f26b45c864e679cade2": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e55e2dfa2f0b4b5ab53eea8e6f4a023f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b909e582846e43fb8380e9916fbc58f0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e1d0477866bc42c783c866c3f4089623": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0aa40681679b4b83a4ebebb24b674575": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b602b7563a254a919e10e7c9bd2e4741": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5bb93adc30a54d62b91c635dfb99c592",
              "IPY_MODEL_c9a9298c522d411ab1cdba78695ea489",
              "IPY_MODEL_76346aa709aa4c96b152e03858adfacf"
            ],
            "layout": "IPY_MODEL_23f12e0039a649c788ad8369bf874d02"
          }
        },
        "5bb93adc30a54d62b91c635dfb99c592": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5c99fcc2a2d9462aac38098432e2e7ca",
            "placeholder": "​",
            "style": "IPY_MODEL_e7aa893a989f4aa9a4cfa7f765f9ffe1",
            "value": "Downloading (…)okenizer_config.json: 100%"
          }
        },
        "c9a9298c522d411ab1cdba78695ea489": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cc07174e0629480197a47fd2307a0c3b",
            "max": 152,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_78712bbc5e434d1e9ea5ea010c526a2f",
            "value": 152
          }
        },
        "76346aa709aa4c96b152e03858adfacf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_9462cdb1e0a34dafaa77377e41d11034",
            "placeholder": "​",
            "style": "IPY_MODEL_c69ca4fd42944054ac52c5456e3ca699",
            "value": " 152/152 [00:00&lt;00:00, 9.37kB/s]"
          }
        },
        "23f12e0039a649c788ad8369bf874d02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5c99fcc2a2d9462aac38098432e2e7ca": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e7aa893a989f4aa9a4cfa7f765f9ffe1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cc07174e0629480197a47fd2307a0c3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "78712bbc5e434d1e9ea5ea010c526a2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9462cdb1e0a34dafaa77377e41d11034": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c69ca4fd42944054ac52c5456e3ca699": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f29e6da16d0f46fea3a55fb32c2aed74": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a7d621bcc0554f6daba3017846f72d8c",
              "IPY_MODEL_3a943251b2a8458fa3ad32eac742795d",
              "IPY_MODEL_5f24c40a6a5d453e97f6d8953d8059d8"
            ],
            "layout": "IPY_MODEL_9ced95ccea2f471d8f644bce1c975c07"
          }
        },
        "a7d621bcc0554f6daba3017846f72d8c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a9607f3c2e2b4903abe34d381ec263cf",
            "placeholder": "​",
            "style": "IPY_MODEL_69dd284006fc48729fa61c55e2ad63aa",
            "value": "Downloading (…)solve/main/vocab.txt: 100%"
          }
        },
        "3a943251b2a8458fa3ad32eac742795d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1edad35d81b54b408d1bc6f50b88d367",
            "max": 213450,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d5a329801cef42f7ae0c3d2206418ba1",
            "value": 213450
          }
        },
        "5f24c40a6a5d453e97f6d8953d8059d8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_8cb1913eb4a540b4a94065b53d40993f",
            "placeholder": "​",
            "style": "IPY_MODEL_af68da6e6c7a45f98b68e9d9310ef82e",
            "value": " 213k/213k [00:00&lt;00:00, 6.58MB/s]"
          }
        },
        "9ced95ccea2f471d8f644bce1c975c07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a9607f3c2e2b4903abe34d381ec263cf": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "69dd284006fc48729fa61c55e2ad63aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1edad35d81b54b408d1bc6f50b88d367": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5a329801cef42f7ae0c3d2206418ba1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8cb1913eb4a540b4a94065b53d40993f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "af68da6e6c7a45f98b68e9d9310ef82e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9afda7a373ca43098faed8a52d475e1d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8be2934aad3a4a478581bf5b3107aeb0",
              "IPY_MODEL_c7ad5922275f4ee3bd045b05d8b19934",
              "IPY_MODEL_ffcb9b0a88d44c84a5d25f36d3555d21"
            ],
            "layout": "IPY_MODEL_961f543ca58f4d268e3de5fe8cbc1338"
          }
        },
        "8be2934aad3a4a478581bf5b3107aeb0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ef3ce859e57747b59e38c06882c220a5",
            "placeholder": "​",
            "style": "IPY_MODEL_2675d077b2d74bc2b39a515831a7a719",
            "value": "Downloading (…)cial_tokens_map.json: 100%"
          }
        },
        "c7ad5922275f4ee3bd045b05d8b19934": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7f286810fa79447fb62e2425ac4b2f15",
            "max": 112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_a6d81dec58784de7829adbb8e3de601e",
            "value": 112
          }
        },
        "ffcb9b0a88d44c84a5d25f36d3555d21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6ad030faf83a402686b6b3c8c8acccd6",
            "placeholder": "​",
            "style": "IPY_MODEL_106e833b1d8c47e1b96f5bd75c321b0e",
            "value": " 112/112 [00:00&lt;00:00, 6.94kB/s]"
          }
        },
        "961f543ca58f4d268e3de5fe8cbc1338": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ef3ce859e57747b59e38c06882c220a5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2675d077b2d74bc2b39a515831a7a719": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7f286810fa79447fb62e2425ac4b2f15": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a6d81dec58784de7829adbb8e3de601e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6ad030faf83a402686b6b3c8c8acccd6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "106e833b1d8c47e1b96f5bd75c321b0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/pranaya-mathur/Deep-Learning-Projects/blob/master/Document_Search_Own_Data_7th_Sept_23.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "g6J8donAmfFh",
        "outputId": "e9beabdf-da7a-4443-a769-a8e8c7b16152"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Requirement already satisfied: pip in /usr/local/lib/python3.10/dist-packages (23.1.2)\n",
            "Collecting pip\n",
            "  Downloading pip-23.2.1-py3-none-any.whl (2.1 MB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 2.1/2.1 MB 12.1 MB/s eta 0:00:00\n",
            "Installing collected packages: pip\n",
            "  Attempting uninstall: pip\n",
            "    Found existing installation: pip 23.1.2\n",
            "    Uninstalling pip-23.1.2:\n",
            "      Successfully uninstalled pip-23.1.2\n",
            "Successfully installed pip-23.2.1\n",
            "Collecting farm-haystack[colab,elasticsearch,inference,preprocessing]\n",
            "  Obtaining dependency information for farm-haystack[colab,elasticsearch,inference,preprocessing] from https://files.pythonhosted.org/packages/a3/e3/8a7ebb563df04300c9e2387ac31e8ffda809b21f276faa838b32cf4a74a7/farm_haystack-1.20.0-py3-none-any.whl.metadata\n",
            "  Downloading farm_haystack-1.20.0-py3-none-any.whl.metadata (25 kB)\n",
            "Collecting boilerpy3 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading boilerpy3-1.0.6-py3-none-any.whl (22 kB)\n",
            "Collecting canals==0.7.0 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for canals==0.7.0 from https://files.pythonhosted.org/packages/08/2f/62e9e455c4cf183ebf6672c47befabb6a36a37a8cdc86256a5632d6ace10/canals-0.7.0-py3-none-any.whl.metadata\n",
            "  Downloading canals-0.7.0-py3-none-any.whl.metadata (4.4 kB)\n",
            "Collecting events (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for events from https://files.pythonhosted.org/packages/25/ed/e47dec0626edd468c84c04d97769e7ab4ea6457b7f54dcb3f72b17fcd876/Events-0.5-py3-none-any.whl.metadata\n",
            "  Downloading Events-0.5-py3-none-any.whl.metadata (3.9 kB)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (4.19.0)\n",
            "Collecting lazy-imports==0.3.1 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading lazy_imports-0.3.1-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (10.1.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.1)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.5.3)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (9.4.0)\n",
            "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.10.0)\n",
            "Collecting posthog (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for posthog from https://files.pythonhosted.org/packages/a7/73/35758818228c70348be4c3c66a76653c62e894e0e3c3461453c5341ca926/posthog-3.0.2-py2.py3-none-any.whl.metadata\n",
            "  Downloading posthog-3.0.2-py2.py3-none-any.whl.metadata (2.0 kB)\n",
            "Collecting prompthub-py==4.0.0 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for prompthub-py==4.0.0 from https://files.pythonhosted.org/packages/27/5f/8c4939e290ff93af79364b88ffe3902d29c234f94e8227cf0b7fce3c887f/prompthub_py-4.0.0-py3-none-any.whl.metadata\n",
            "  Downloading prompthub_py-4.0.0-py3-none-any.whl.metadata (2.2 kB)\n",
            "Collecting pydantic<2 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for pydantic<2 from https://files.pythonhosted.org/packages/bc/e0/0371e9b6c910afe502e5fe18cc94562bfd9399617c7b4f5b6e13c29115b3/pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
            "  Downloading pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (149 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 149.3/149.3 kB 6.2 MB/s eta 0:00:00\n",
            "Collecting quantulum3 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading quantulum3-0.9.0-py3-none-any.whl (10.7 MB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.7/10.7 MB 32.4 MB/s eta 0:00:00\n",
            "Collecting rank-bm25 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading rank_bm25-0.2.2-py3-none-any.whl (8.6 kB)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (2.31.0)\n",
            "Collecting requests-cache<1.0.0 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading requests_cache-0.9.8-py3-none-any.whl (48 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 48.7/48.7 kB 6.5 MB/s eta 0:00:00\n",
            "Collecting scikit-learn>=1.3.0 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for scikit-learn>=1.3.0 from https://files.pythonhosted.org/packages/5c/e9/ee572691a3fb05555bcde41826faad29ae4bc1fb07982e7f53d54a176879/scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
            "  Downloading scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (11 kB)\n",
            "Collecting sseclient-py (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for sseclient-py from https://files.pythonhosted.org/packages/49/58/97655efdfeb5b4eeab85b1fc5d3fa1023661246c2ab2a26ea8e47402d4f2/sseclient_py-1.8.0-py2.py3-none-any.whl.metadata\n",
            "  Downloading sseclient_py-1.8.0-py2.py3-none-any.whl.metadata (2.0 kB)\n",
            "Requirement already satisfied: tenacity in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (8.2.3)\n",
            "Collecting tiktoken>=0.3.2 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading tiktoken-0.4.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.7 MB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.7/1.7 MB 60.1 MB/s eta 0:00:00\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (4.66.1)\n",
            "Collecting transformers==4.32.1 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for transformers==4.32.1 from https://files.pythonhosted.org/packages/83/8d/f65f8138365462ace54458a9e164f4b28ce1141361970190eef36bdef986/transformers-4.32.1-py3-none-any.whl.metadata\n",
            "  Downloading transformers-4.32.1-py3-none-any.whl.metadata (118 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 118.5/118.5 kB 16.6 MB/s eta 0:00:00\n",
            "Collecting langdetect (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading langdetect-1.0.9.tar.gz (981 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 981.5/981.5 kB 56.2 MB/s eta 0:00:00\n",
            "  Preparing metadata (setup.py): started\n",
            "  Preparing metadata (setup.py): finished with status 'done'\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (from farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.8.1)\n",
            "Collecting pillow (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading Pillow-9.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (4.3 MB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 4.3/4.3 MB 80.0 MB/s eta 0:00:00\n",
            "Collecting huggingface-hub>=0.5.0 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for huggingface-hub>=0.5.0 from https://files.pythonhosted.org/packages/7f/c4/adcbe9a696c135578cabcbdd7331332daad4d49b7c43688bc2d36b3a47d2/huggingface_hub-0.16.4-py3-none-any.whl.metadata\n",
            "  Downloading huggingface_hub-0.16.4-py3-none-any.whl.metadata (12 kB)\n",
            "Collecting sentence-transformers>=2.2.0 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading sentence-transformers-2.2.2.tar.gz (85 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 86.0/86.0 kB 12.1 MB/s eta 0:00:00\n",
            "  Preparing metadata (setup.py): started\n",
            "  Preparing metadata (setup.py): finished with status 'done'\n",
            "Requirement already satisfied: pyyaml<7.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from prompthub-py==4.0.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (6.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.12.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (23.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2023.6.3)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1 (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading tokenizers-0.13.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.8 MB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.8/7.8 MB 97.3 MB/s eta 0:00:00\n",
            "Collecting safetensors>=0.3.1 (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for safetensors>=0.3.1 from https://files.pythonhosted.org/packages/6c/f0/c17bbdb1e5f9dab29d44cade445135789f75f8f08ea2728d04493ea8412b/safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata\n",
            "  Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (4.7 kB)\n",
            "Requirement already satisfied: torch!=1.12.0,>=1.9 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2.0.1+cu118)\n",
            "Collecting accelerate>=0.20.3 (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for accelerate>=0.20.3 from https://files.pythonhosted.org/packages/4d/a7/05c67003d659a0035f2b3a8cf389c1d9645865aee84a73ce99ddab16682f/accelerate-0.22.0-py3-none-any.whl.metadata\n",
            "  Downloading accelerate-0.22.0-py3-none-any.whl.metadata (17 kB)\n",
            "Collecting sentencepiece!=0.1.92,>=0.1.91 (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading sentencepiece-0.1.99-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 88.3 MB/s eta 0:00:00\n",
            "Requirement already satisfied: protobuf in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.20.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.5.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2023.6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.10/dist-packages (from huggingface-hub>=0.5.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (4.7.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2023.7.22)\n",
            "Requirement already satisfied: appdirs>=1.4.4 in /usr/local/lib/python3.10/dist-packages (from requests-cache<1.0.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.4.4)\n",
            "Requirement already satisfied: attrs>=21.2 in /usr/local/lib/python3.10/dist-packages (from requests-cache<1.0.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (23.1.0)\n",
            "Collecting cattrs>=22.2 (from requests-cache<1.0.0->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for cattrs>=22.2 from https://files.pythonhosted.org/packages/3a/ba/05df14efaa0624fac6b1510e87f5ce446208d2f6ce50270a89b6268aebfe/cattrs-23.1.2-py3-none-any.whl.metadata\n",
            "  Downloading cattrs-23.1.2-py3-none-any.whl.metadata (9.3 kB)\n",
            "Collecting url-normalize>=1.4 (from requests-cache<1.0.0->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading url_normalize-1.4.3-py2.py3-none-any.whl (6.8 kB)\n",
            "Requirement already satisfied: scipy>=1.5.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.3.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.10.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.3.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.3.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.2.0)\n",
            "Requirement already satisfied: torchvision in /usr/local/lib/python3.10/dist-packages (from sentence-transformers>=2.2.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (0.15.2+cu118)\n",
            "Collecting elastic-transport<8 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading elastic_transport-7.16.0-py2.py3-none-any.whl (35 kB)\n",
            "Collecting elasticsearch<8,>=7.17 (from farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading elasticsearch-7.17.9-py2.py3-none-any.whl (385 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 386.0/386.0 kB 23.3 MB/s eta 0:00:00\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->farm-haystack[colab,elasticsearch,inference,preprocessing]) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->farm-haystack[colab,elasticsearch,inference,preprocessing]) (0.10.2)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.10/dist-packages (from langdetect->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk->farm-haystack[colab,elasticsearch,inference,preprocessing]) (8.1.7)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2023.3.post1)\n",
            "Collecting monotonic>=1.5 (from posthog->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading monotonic-1.6-py2.py3-none-any.whl (8.2 kB)\n",
            "Collecting backoff>=1.10.0 (from posthog->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading backoff-2.2.1-py3-none-any.whl (15 kB)\n",
            "Requirement already satisfied: inflect in /usr/local/lib/python3.10/dist-packages (from quantulum3->farm-haystack[colab,elasticsearch,inference,preprocessing]) (7.0.0)\n",
            "Collecting num2words (from quantulum3->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading num2words-0.5.12-py3-none-any.whl (125 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 125.2/125.2 kB 15.9 MB/s eta 0:00:00\n",
            "Requirement already satisfied: psutil in /usr/local/lib/python3.10/dist-packages (from accelerate>=0.20.3->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (5.9.5)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from cattrs>=22.2->requests-cache<1.0.0->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.1.3)\n",
            "Collecting urllib3<3,>=1.21.1 (from requests->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Obtaining dependency information for urllib3<3,>=1.21.1 from https://files.pythonhosted.org/packages/c5/05/c214b32d21c0b465506f95c4f28ccbcba15022e000b043b72b3df7728471/urllib3-1.26.16-py2.py3-none-any.whl.metadata\n",
            "  Downloading urllib3-1.26.16-py2.py3-none-any.whl.metadata (48 kB)\n",
            "     ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 48.4/48.4 kB 6.2 MB/s eta 0:00:00\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.9->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.12)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.9->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.1.2)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from torch!=1.12.0,>=1.9->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2.0.0)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.12.0,>=1.9->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (3.27.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->torch!=1.12.0,>=1.9->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (16.0.6)\n",
            "Collecting docopt>=0.6.2 (from num2words->quantulum3->farm-haystack[colab,elasticsearch,inference,preprocessing])\n",
            "  Downloading docopt-0.6.2.tar.gz (25 kB)\n",
            "  Preparing metadata (setup.py): started\n",
            "  Preparing metadata (setup.py): finished with status 'done'\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->torch!=1.12.0,>=1.9->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (2.1.3)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch!=1.12.0,>=1.9->transformers==4.32.1->farm-haystack[colab,elasticsearch,inference,preprocessing]) (1.3.0)\n",
            "Downloading canals-0.7.0-py3-none-any.whl (32 kB)\n",
            "Downloading prompthub_py-4.0.0-py3-none-any.whl (6.9 kB)\n",
            "Downloading transformers-4.32.1-py3-none-any.whl (7.5 MB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 7.5/7.5 MB 94.4 MB/s eta 0:00:00\n",
            "Downloading huggingface_hub-0.16.4-py3-none-any.whl (268 kB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 268.8/268.8 kB 31.2 MB/s eta 0:00:00\n",
            "Downloading pydantic-1.10.12-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (3.1 MB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 3.1/3.1 MB 84.2 MB/s eta 0:00:00\n",
            "Downloading scikit_learn-1.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (10.8 MB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 10.8/10.8 MB 88.8 MB/s eta 0:00:00\n",
            "Downloading Events-0.5-py3-none-any.whl (6.8 kB)\n",
            "Downloading farm_haystack-1.20.0-py3-none-any.whl (789 kB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 789.9/789.9 kB 64.0 MB/s eta 0:00:00\n",
            "Downloading posthog-3.0.2-py2.py3-none-any.whl (37 kB)\n",
            "Downloading sseclient_py-1.8.0-py2.py3-none-any.whl (8.8 kB)\n",
            "Downloading accelerate-0.22.0-py3-none-any.whl (251 kB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 251.2/251.2 kB 34.7 MB/s eta 0:00:00\n",
            "Downloading cattrs-23.1.2-py3-none-any.whl (50 kB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 50.8/50.8 kB 7.6 MB/s eta 0:00:00\n",
            "Downloading safetensors-0.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.3 MB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 1.3/1.3 MB 88.6 MB/s eta 0:00:00\n",
            "Downloading urllib3-1.26.16-py2.py3-none-any.whl (143 kB)\n",
            "   ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ 143.1/143.1 kB 20.1 MB/s eta 0:00:00\n",
            "Building wheels for collected packages: sentence-transformers, langdetect, docopt\n",
            "  Building wheel for sentence-transformers (setup.py): started\n",
            "  Building wheel for sentence-transformers (setup.py): finished with status 'done'\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-2.2.2-py3-none-any.whl size=125923 sha256=2d617982601259a6f95be549cbf503ca913c01d882e5bb956858e3700560ba45\n",
            "  Stored in directory: /root/.cache/pip/wheels/62/f2/10/1e606fd5f02395388f74e7462910fe851042f97238cbbd902f\n",
            "  Building wheel for langdetect (setup.py): started\n",
            "  Building wheel for langdetect (setup.py): finished with status 'done'\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-py3-none-any.whl size=993224 sha256=1017989bde44bc6064fc10bc94513ebb41841306d4b81fe532851b9f3decc192\n",
            "  Stored in directory: /root/.cache/pip/wheels/95/03/7d/59ea870c70ce4e5a370638b5462a7711ab78fba2f655d05106\n",
            "  Building wheel for docopt (setup.py): started\n",
            "  Building wheel for docopt (setup.py): finished with status 'done'\n",
            "  Created wheel for docopt: filename=docopt-0.6.2-py2.py3-none-any.whl size=13705 sha256=53e92d353387b530ee2fd8c266ee81cbb7496dd682a5cf0ee89220c8ab0be2da\n",
            "  Stored in directory: /root/.cache/pip/wheels/fc/ab/d4/5da2067ac95b36618c629a5f93f809425700506f72c9732fac\n",
            "Successfully built sentence-transformers langdetect docopt\n",
            "Installing collected packages: tokenizers, sseclient-py, sentencepiece, safetensors, monotonic, events, docopt, urllib3, url-normalize, rank-bm25, pydantic, pillow, num2words, lazy-imports, langdetect, cattrs, canals, boilerpy3, backoff, scikit-learn, elasticsearch, elastic-transport, tiktoken, requests-cache, quantulum3, prompthub-py, posthog, huggingface-hub, transformers, farm-haystack, accelerate, sentence-transformers\n",
            "  Attempting uninstall: urllib3\n",
            "    Found existing installation: urllib3 2.0.4\n",
            "    Uninstalling urllib3-2.0.4:\n",
            "      Successfully uninstalled urllib3-2.0.4\n",
            "  Attempting uninstall: pydantic\n",
            "    Found existing installation: pydantic 2.3.0\n",
            "    Uninstalling pydantic-2.3.0:\n",
            "      Successfully uninstalled pydantic-2.3.0\n",
            "  Attempting uninstall: pillow\n",
            "    Found existing installation: Pillow 9.4.0\n",
            "    Uninstalling Pillow-9.4.0:\n",
            "      Successfully uninstalled Pillow-9.4.0\n",
            "  Attempting uninstall: scikit-learn\n",
            "    Found existing installation: scikit-learn 1.2.2\n",
            "    Uninstalling scikit-learn-1.2.2:\n",
            "      Successfully uninstalled scikit-learn-1.2.2\n",
            "Successfully installed accelerate-0.22.0 backoff-2.2.1 boilerpy3-1.0.6 canals-0.7.0 cattrs-23.1.2 docopt-0.6.2 elastic-transport-7.16.0 elasticsearch-7.17.9 events-0.5 farm-haystack-1.20.0 huggingface-hub-0.16.4 langdetect-1.0.9 lazy-imports-0.3.1 monotonic-1.6 num2words-0.5.12 pillow-9.0.0 posthog-3.0.2 prompthub-py-4.0.0 pydantic-1.10.12 quantulum3-0.9.0 rank-bm25-0.2.2 requests-cache-0.9.8 safetensors-0.3.3 scikit-learn-1.3.0 sentence-transformers-2.2.2 sentencepiece-0.1.99 sseclient-py-1.8.0 tiktoken-0.4.0 tokenizers-0.13.3 transformers-4.32.1 url-normalize-1.4.3 urllib3-1.26.16\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\n"
          ]
        }
      ],
      "source": [
        "%%bash\n",
        "\n",
        "pip install --upgrade pip\n",
        "pip install farm-haystack[colab,preprocessing,elasticsearch,inference]"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash\n",
        "\n",
        "wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q\n",
        "tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz\n",
        "chown -R daemon:daemon elasticsearch-7.9.2"
      ],
      "metadata": {
        "id": "-RE5Wap2nYMh"
      },
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "%%bash --bg\n",
        "\n",
        "sudo -u daemon -- elasticsearch-7.9.2/bin/elasticsearch"
      ],
      "metadata": {
        "id": "ooLXzbC6ndaa"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import time\n",
        "\n",
        "time.sleep(30)"
      ],
      "metadata": {
        "id": "6DwESCVrnfLd"
      },
      "execution_count": 4,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import os\n",
        "from haystack.document_stores import ElasticsearchDocumentStore\n",
        "\n",
        "# Get the host where Elasticsearch is running, default to localhost\n",
        "host = os.environ.get(\"ELASTICSEARCH_HOST\", \"localhost\")\n",
        "\n",
        "document_store = ElasticsearchDocumentStore(host=host, username=\"\", password=\"\", index='aurelius')"
      ],
      "metadata": {
        "id": "vQ27G4PMnrJl"
      },
      "execution_count": 5,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import requests\n",
        "data = requests.get('https://raw.githubusercontent.com/jamescalam/transformers/main/data/text/meditations/clean.txt')\n",
        "text = data.text.split('\\n')"
      ],
      "metadata": {
        "id": "h8mvem5AnxiV"
      },
      "execution_count": 6,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "text[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_TnreP0gn8Zk",
        "outputId": "0b9d759f-7c15-41d9-f451-98afae47b0c9"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['From my grandfather Verus I learned good morals and the government of my temper.',\n",
              " 'From the reputation and remembrance of my father, modesty and a manly character.',\n",
              " 'From my mother, piety and beneficence, and abstinence, not only from evil deeds, but even from evil thoughts; and further, simplicity in my way of living, far removed from the habits of the rich.']"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "requests.get('http://localhost:9200/_cluster/health').json()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "s_GrvAq8n-WZ",
        "outputId": "3e1581ce-f6ba-4477-eac3-bfbcb89b9e8a"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'cluster_name': 'elasticsearch',\n",
              " 'status': 'yellow',\n",
              " 'timed_out': False,\n",
              " 'number_of_nodes': 1,\n",
              " 'number_of_data_nodes': 1,\n",
              " 'active_primary_shards': 2,\n",
              " 'active_shards': 2,\n",
              " 'relocating_shards': 0,\n",
              " 'initializing_shards': 0,\n",
              " 'unassigned_shards': 2,\n",
              " 'delayed_unassigned_shards': 0,\n",
              " 'number_of_pending_tasks': 0,\n",
              " 'number_of_in_flight_fetch': 0,\n",
              " 'task_max_waiting_in_queue_millis': 0,\n",
              " 'active_shards_percent_as_number': 50.0}"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(requests.get('http://localhost:9200/_cat/indices').text)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2K2N2iqNoAZ_",
        "outputId": "4b753fa8-b98f-4a36-a82a-5376c02a17ba"
      },
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "yellow open aurelius B9T4fiB0Rti3rGeLDQoCsw 1 1 0 0 208b 208b\n",
            "yellow open label    -6p5Ycv7Sb6TJS5OwS75Ig 1 1 0 0 208b 208b\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "data_json = [\n",
        "    {\n",
        "        'content': paragraph,\n",
        "        'meta': {\n",
        "            'source': 'meditations'\n",
        "        }\n",
        "    } for paragraph in text\n",
        "]"
      ],
      "metadata": {
        "id": "bIolIBqcoDmv"
      },
      "execution_count": 10,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "data_json[:3]"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MPulkTfloOIy",
        "outputId": "da2b47ae-6be5-41b4-97fc-ad90b1a95d5c"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[{'content': 'From my grandfather Verus I learned good morals and the government of my temper.',\n",
              "  'meta': {'source': 'meditations'}},\n",
              " {'content': 'From the reputation and remembrance of my father, modesty and a manly character.',\n",
              "  'meta': {'source': 'meditations'}},\n",
              " {'content': 'From my mother, piety and beneficence, and abstinence, not only from evil deeds, but even from evil thoughts; and further, simplicity in my way of living, far removed from the habits of the rich.',\n",
              "  'meta': {'source': 'meditations'}}]"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "len(data_json)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RpKQQXFIoRYj",
        "outputId": "bacf4f86-0c37-4842-94fa-2f0c5cd66136"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "507"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "document_store.write_documents(data_json)"
      ],
      "metadata": {
        "id": "x_UvNzVSoTfe"
      },
      "execution_count": 14,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "requests.get('http://localhost:9200/aurelius/_count').json()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BLR14iu0oZvI",
        "outputId": "48364285-7715-4c7d-ee9d-bd3f64358e37"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'count': 507,\n",
              " '_shards': {'total': 1, 'successful': 1, 'skipped': 0, 'failed': 0}}"
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from haystack.nodes import BM25Retriever\n",
        "# from haystack.reader.farm import FARMReader\n",
        "\n",
        "# retriever = BM25Retriever(document_store)  # BM25\n",
        "# reader = FARMReader(model_name_or_path='deepset/bert-base-cased-squad2',\n",
        "#                     context_window_size=1500,\n",
        "#                     use_gpu=True)"
      ],
      "metadata": {
        "id": "lvQYT8sMogj7"
      },
      "execution_count": 17,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "!pip install git+https://github.com/deepset-ai/haystack.git"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "slPQq-Ykpn8Y",
        "outputId": "d2963697-8d1d-41d8-ab80-c62da3517f8f"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Collecting git+https://github.com/deepset-ai/haystack.git\n",
            "  Cloning https://github.com/deepset-ai/haystack.git to /tmp/pip-req-build-r_4lz3wd\n",
            "  Running command git clone --filter=blob:none --quiet https://github.com/deepset-ai/haystack.git /tmp/pip-req-build-r_4lz3wd\n",
            "  Resolved https://github.com/deepset-ai/haystack.git to commit b7bea3ae9c59a5e2bd2228a355bcc562b3f80ba6\n",
            "  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: boilerpy3 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (1.0.6)\n",
            "Collecting canals==0.8.0 (from farm-haystack==1.21.0rc0)\n",
            "  Obtaining dependency information for canals==0.8.0 from https://files.pythonhosted.org/packages/92/63/d0f20dd35a3f770baf4359f79635d7fb723ea9bf88800644fbf260c0bd4a/canals-0.8.0-py3-none-any.whl.metadata\n",
            "  Downloading canals-0.8.0-py3-none-any.whl.metadata (4.4 kB)\n",
            "Requirement already satisfied: events in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (0.5)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (3.1.2)\n",
            "Requirement already satisfied: jsonschema in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (4.19.0)\n",
            "Requirement already satisfied: lazy-imports==0.3.1 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (0.3.1)\n",
            "Requirement already satisfied: more-itertools in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (10.1.0)\n",
            "Requirement already satisfied: networkx in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (3.1)\n",
            "Collecting openai (from farm-haystack==1.21.0rc0)\n",
            "  Obtaining dependency information for openai from https://files.pythonhosted.org/packages/ae/59/911d6e5f1d7514d79c527067643376cddcf4cb8d1728e599b3b03ab51c69/openai-0.28.0-py3-none-any.whl.metadata\n",
            "  Downloading openai-0.28.0-py3-none-any.whl.metadata (13 kB)\n",
            "Collecting openai-whisper (from farm-haystack==1.21.0rc0)\n",
            "  Downloading openai-whisper-20230314.tar.gz (792 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m792.9/792.9 kB\u001b[0m \u001b[31m9.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (1.5.3)\n",
            "Requirement already satisfied: pillow in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (9.0.0)\n",
            "Requirement already satisfied: platformdirs in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (3.10.0)\n",
            "Requirement already satisfied: posthog in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (3.0.2)\n",
            "Requirement already satisfied: prompthub-py==4.0.0 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (4.0.0)\n",
            "Requirement already satisfied: pydantic<2 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (1.10.12)\n",
            "Requirement already satisfied: quantulum3 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (0.9.0)\n",
            "Requirement already satisfied: rank-bm25 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (0.2.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (2.31.0)\n",
            "Requirement already satisfied: requests-cache<1.0.0 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (0.9.8)\n",
            "Requirement already satisfied: scikit-learn>=1.3.0 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (1.3.0)\n",
            "Requirement already satisfied: sseclient-py in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (1.8.0)\n",
            "Requirement already satisfied: tenacity in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (8.2.3)\n",
            "Requirement already satisfied: tiktoken>=0.3.2 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (0.4.0)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (4.66.1)\n",
            "Requirement already satisfied: transformers==4.32.1 in /usr/local/lib/python3.10/dist-packages (from farm-haystack==1.21.0rc0) (4.32.1)\n",
            "Requirement already satisfied: pyyaml<7.0,>=6.0 in /usr/local/lib/python3.10/dist-packages (from prompthub-py==4.0.0->farm-haystack==1.21.0rc0) (6.0.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack==1.21.0rc0) (3.12.3)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.15.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack==1.21.0rc0) (0.16.4)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack==1.21.0rc0) (1.23.5)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack==1.21.0rc0) (23.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack==1.21.0rc0) (2023.6.3)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack==1.21.0rc0) (0.13.3)\n",
            "Requirement already satisfied: safetensors>=0.3.1 in /usr/local/lib/python3.10/dist-packages (from transformers==4.32.1->farm-haystack==1.21.0rc0) (0.3.3)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<2->farm-haystack==1.21.0rc0) (4.7.1)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack==1.21.0rc0) (3.2.0)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack==1.21.0rc0) (3.4)\n",
            "Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack==1.21.0rc0) (1.26.16)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests->farm-haystack==1.21.0rc0) (2023.7.22)\n",
            "Requirement already satisfied: appdirs>=1.4.4 in /usr/local/lib/python3.10/dist-packages (from requests-cache<1.0.0->farm-haystack==1.21.0rc0) (1.4.4)\n",
            "Requirement already satisfied: attrs>=21.2 in /usr/local/lib/python3.10/dist-packages (from requests-cache<1.0.0->farm-haystack==1.21.0rc0) (23.1.0)\n",
            "Requirement already satisfied: cattrs>=22.2 in /usr/local/lib/python3.10/dist-packages (from requests-cache<1.0.0->farm-haystack==1.21.0rc0) (23.1.2)\n",
            "Requirement already satisfied: url-normalize>=1.4 in /usr/local/lib/python3.10/dist-packages (from requests-cache<1.0.0->farm-haystack==1.21.0rc0) (1.4.3)\n",
            "Requirement already satisfied: scipy>=1.5.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.3.0->farm-haystack==1.21.0rc0) (1.10.1)\n",
            "Requirement already satisfied: joblib>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.3.0->farm-haystack==1.21.0rc0) (1.3.2)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.10/dist-packages (from scikit-learn>=1.3.0->farm-haystack==1.21.0rc0) (3.2.0)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->farm-haystack==1.21.0rc0) (2.1.3)\n",
            "Requirement already satisfied: jsonschema-specifications>=2023.03.6 in /usr/local/lib/python3.10/dist-packages (from jsonschema->farm-haystack==1.21.0rc0) (2023.7.1)\n",
            "Requirement already satisfied: referencing>=0.28.4 in /usr/local/lib/python3.10/dist-packages (from jsonschema->farm-haystack==1.21.0rc0) (0.30.2)\n",
            "Requirement already satisfied: rpds-py>=0.7.1 in /usr/local/lib/python3.10/dist-packages (from jsonschema->farm-haystack==1.21.0rc0) (0.10.2)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.10/dist-packages (from openai->farm-haystack==1.21.0rc0) (3.8.5)\n",
            "Requirement already satisfied: triton==2.0.0 in /usr/local/lib/python3.10/dist-packages (from openai-whisper->farm-haystack==1.21.0rc0) (2.0.0)\n",
            "Requirement already satisfied: numba in /usr/local/lib/python3.10/dist-packages (from openai-whisper->farm-haystack==1.21.0rc0) (0.56.4)\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.10/dist-packages (from openai-whisper->farm-haystack==1.21.0rc0) (2.0.1+cu118)\n",
            "INFO: pip is looking at multiple versions of openai-whisper to determine which version is compatible with other requirements. This could take a while.\n",
            "  Downloading openai-whisper-20230308.tar.gz (1.2 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.2/1.2 MB\u001b[0m \u001b[31m17.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Installing build dependencies ... \u001b[?25l\u001b[?25hdone\n",
            "  Getting requirements to build wheel ... \u001b[?25l\u001b[?25hdone\n",
            "  Preparing metadata (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "Collecting ffmpeg-python==0.2.0 (from openai-whisper->farm-haystack==1.21.0rc0)\n",
            "  Downloading ffmpeg_python-0.2.0-py3-none-any.whl (25 kB)\n",
            "Requirement already satisfied: future in /usr/local/lib/python3.10/dist-packages (from ffmpeg-python==0.2.0->openai-whisper->farm-haystack==1.21.0rc0) (0.18.3)\n",
            "Requirement already satisfied: cmake in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->openai-whisper->farm-haystack==1.21.0rc0) (3.27.2)\n",
            "Requirement already satisfied: lit in /usr/local/lib/python3.10/dist-packages (from triton==2.0.0->openai-whisper->farm-haystack==1.21.0rc0) (16.0.6)\n",
            "Requirement already satisfied: python-dateutil>=2.8.1 in /usr/local/lib/python3.10/dist-packages (from pandas->farm-haystack==1.21.0rc0) (2.8.2)\n",
            "Requirement already satisfied: pytz>=2020.1 in /usr/local/lib/python3.10/dist-packages (from pandas->farm-haystack==1.21.0rc0) (2023.3.post1)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog->farm-haystack==1.21.0rc0) (1.16.0)\n",
            "Requirement already satisfied: monotonic>=1.5 in /usr/local/lib/python3.10/dist-packages (from posthog->farm-haystack==1.21.0rc0) (1.6)\n",
            "Requirement already satisfied: backoff>=1.10.0 in /usr/local/lib/python3.10/dist-packages (from posthog->farm-haystack==1.21.0rc0) (2.2.1)\n",
            "Requirement already satisfied: inflect in /usr/local/lib/python3.10/dist-packages (from quantulum3->farm-haystack==1.21.0rc0) (7.0.0)\n",
            "Requirement already satisfied: num2words in /usr/local/lib/python3.10/dist-packages (from quantulum3->farm-haystack==1.21.0rc0) (0.5.12)\n",
            "Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from cattrs>=22.2->requests-cache<1.0.0->farm-haystack==1.21.0rc0) (1.1.3)\n",
            "Requirement already satisfied: fsspec in /usr/local/lib/python3.10/dist-packages (from huggingface-hub<1.0,>=0.15.1->transformers==4.32.1->farm-haystack==1.21.0rc0) (2023.6.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai->farm-haystack==1.21.0rc0) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai->farm-haystack==1.21.0rc0) (4.0.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai->farm-haystack==1.21.0rc0) (1.9.2)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai->farm-haystack==1.21.0rc0) (1.4.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp->openai->farm-haystack==1.21.0rc0) (1.3.1)\n",
            "Requirement already satisfied: docopt>=0.6.2 in /usr/local/lib/python3.10/dist-packages (from num2words->quantulum3->farm-haystack==1.21.0rc0) (0.6.2)\n",
            "Requirement already satisfied: llvmlite<0.40,>=0.39.0dev0 in /usr/local/lib/python3.10/dist-packages (from numba->openai-whisper->farm-haystack==1.21.0rc0) (0.39.1)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from numba->openai-whisper->farm-haystack==1.21.0rc0) (67.7.2)\n",
            "Requirement already satisfied: sympy in /usr/local/lib/python3.10/dist-packages (from torch->openai-whisper->farm-haystack==1.21.0rc0) (1.12)\n",
            "Requirement already satisfied: mpmath>=0.19 in /usr/local/lib/python3.10/dist-packages (from sympy->torch->openai-whisper->farm-haystack==1.21.0rc0) (1.3.0)\n",
            "Downloading canals-0.8.0-py3-none-any.whl (33 kB)\n",
            "Downloading openai-0.28.0-py3-none-any.whl (76 kB)\n",
            "\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.5/76.5 kB\u001b[0m \u001b[31m11.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hBuilding wheels for collected packages: farm-haystack, openai-whisper\n",
            "  Building wheel for farm-haystack (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for farm-haystack: filename=farm_haystack-1.21.0rc0-py3-none-any.whl size=802207 sha256=f6200409ccec8811f9695368f612e9dfd30d4e588feb4b931e6f2664f990a0f4\n",
            "  Stored in directory: /tmp/pip-ephem-wheel-cache-s_gy7_j9/wheels/e7/f8/df/e7c3a1fea47ea9a9db18c056e698a82658e0fe5814c5608165\n",
            "  Building wheel for openai-whisper (pyproject.toml) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for openai-whisper: filename=openai_whisper-20230308-py3-none-any.whl size=1187477 sha256=ed8d8cc7ef5b651af56f6a2729b598e785c0b71c622e10cb25ef5527d283aa5f\n",
            "  Stored in directory: /root/.cache/pip/wheels/7d/f8/3b/8b9191e3ba805fe7459b9315247fdaf2cd663a9feda760e2df\n",
            "Successfully built farm-haystack openai-whisper\n",
            "Installing collected packages: ffmpeg-python, canals, openai, openai-whisper, farm-haystack\n",
            "  Attempting uninstall: canals\n",
            "    Found existing installation: canals 0.7.0\n",
            "    Uninstalling canals-0.7.0:\n",
            "      Successfully uninstalled canals-0.7.0\n",
            "  Attempting uninstall: farm-haystack\n",
            "    Found existing installation: farm-haystack 1.20.0\n",
            "    Uninstalling farm-haystack-1.20.0:\n",
            "      Successfully uninstalled farm-haystack-1.20.0\n",
            "Successfully installed canals-0.8.0 farm-haystack-1.21.0rc0 ffmpeg-python-0.2.0 openai-0.28.0 openai-whisper-20230308\n",
            "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
            "\u001b[0m"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from haystack.nodes import FARMReader\n",
        "\n",
        "retriever = BM25Retriever(document_store)  # BM25\n",
        "reader = FARMReader(model_name_or_path='deepset/bert-base-cased-squad2',\n",
        "                    context_window_size=1500,\n",
        "                    use_gpu=True)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 249,
          "referenced_widgets": [
            "a04e2e89bfb1494d82cfba09c596c1bf",
            "b3db777fee414a78b15c8a3c1dc7767a",
            "6cd0fbb46203482189c0b418ca94b325",
            "03e1fb810bbd4ca4b1956a8ec4ccdad1",
            "13217588ae194d438d486de317f6eecb",
            "42edf9a594dc4a6884e7d1171b6c480c",
            "60407a2a68e640429525423e50ea9c31",
            "3952e294794d461db461c4924191aef5",
            "6b22b383941644cc8560be08e6072bac",
            "0c92bc3d4b894c8ca597253b7c4cde13",
            "9efd3aac223146559b27191a80c3bc1f",
            "71bad721f024403b87a8379de152571a",
            "927efe7a40314379a659d807b1be44bb",
            "5d41705b6a694a0baa31bba07c9f4a10",
            "372fc7aa0a5e4d53956677f807f2f497",
            "aaf7b5905af04b6e97acf1b4dbe14aeb",
            "a55f64e1336b4558bdc5cfb3c45b65cd",
            "6b913c98d7774f26b45c864e679cade2",
            "e55e2dfa2f0b4b5ab53eea8e6f4a023f",
            "b909e582846e43fb8380e9916fbc58f0",
            "e1d0477866bc42c783c866c3f4089623",
            "0aa40681679b4b83a4ebebb24b674575",
            "b602b7563a254a919e10e7c9bd2e4741",
            "5bb93adc30a54d62b91c635dfb99c592",
            "c9a9298c522d411ab1cdba78695ea489",
            "76346aa709aa4c96b152e03858adfacf",
            "23f12e0039a649c788ad8369bf874d02",
            "5c99fcc2a2d9462aac38098432e2e7ca",
            "e7aa893a989f4aa9a4cfa7f765f9ffe1",
            "cc07174e0629480197a47fd2307a0c3b",
            "78712bbc5e434d1e9ea5ea010c526a2f",
            "9462cdb1e0a34dafaa77377e41d11034",
            "c69ca4fd42944054ac52c5456e3ca699",
            "f29e6da16d0f46fea3a55fb32c2aed74",
            "a7d621bcc0554f6daba3017846f72d8c",
            "3a943251b2a8458fa3ad32eac742795d",
            "5f24c40a6a5d453e97f6d8953d8059d8",
            "9ced95ccea2f471d8f644bce1c975c07",
            "a9607f3c2e2b4903abe34d381ec263cf",
            "69dd284006fc48729fa61c55e2ad63aa",
            "1edad35d81b54b408d1bc6f50b88d367",
            "d5a329801cef42f7ae0c3d2206418ba1",
            "8cb1913eb4a540b4a94065b53d40993f",
            "af68da6e6c7a45f98b68e9d9310ef82e",
            "9afda7a373ca43098faed8a52d475e1d",
            "8be2934aad3a4a478581bf5b3107aeb0",
            "c7ad5922275f4ee3bd045b05d8b19934",
            "ffcb9b0a88d44c84a5d25f36d3555d21",
            "961f543ca58f4d268e3de5fe8cbc1338",
            "ef3ce859e57747b59e38c06882c220a5",
            "2675d077b2d74bc2b39a515831a7a719",
            "7f286810fa79447fb62e2425ac4b2f15",
            "a6d81dec58784de7829adbb8e3de601e",
            "6ad030faf83a402686b6b3c8c8acccd6",
            "106e833b1d8c47e1b96f5bd75c321b0e"
          ]
        },
        "id": "ONNuVqhXpCIa",
        "outputId": "4050b1d4-f95e-4a84-947c-62fbbbebf7bc"
      },
      "execution_count": 21,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)lve/main/config.json:   0%|          | 0.00/508 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a04e2e89bfb1494d82cfba09c596c1bf"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading model.safetensors:   0%|          | 0.00/433M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "71bad721f024403b87a8379de152571a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at deepset/bert-base-cased-squad2 were not used when initializing BertForQuestionAnswering: ['bert.pooler.dense.weight', 'bert.pooler.dense.bias']\n",
            "- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)okenizer_config.json:   0%|          | 0.00/152 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "b602b7563a254a919e10e7c9bd2e4741"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)solve/main/vocab.txt:   0%|          | 0.00/213k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "f29e6da16d0f46fea3a55fb32c2aed74"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading (…)cial_tokens_map.json:   0%|          | 0.00/112 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9afda7a373ca43098faed8a52d475e1d"
            }
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from haystack.pipelines import ExtractiveQAPipeline\n",
        "\n",
        "qa = ExtractiveQAPipeline(reader=reader, retriever=retriever)"
      ],
      "metadata": {
        "id": "QHqfbHvDpcTi"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "qa.run(query='What did your grandfather teach?')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VY8uHqy0qOSg",
        "outputId": "2030bbe5-b424-4c35-c746-5262bbf8d7db"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Inferencing Samples: 100%|██████████| 1/1 [00:03<00:00,  3.62s/ Batches]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'query': 'What did your grandfather teach?',\n",
              " 'no_ans_gap': -3.121262550354004,\n",
              " 'answers': [<Answer {'answer': 'good morals and the government of my temper', 'type': 'extractive', 'score': 0.16657456755638123, 'context': 'From my grandfather Verus I learned good morals and the government of my temper.', 'offsets_in_document': [{'start': 36, 'end': 79}], 'offsets_in_context': [{'start': 36, 'end': 79}], 'document_ids': ['ecc466cb76ed8dedab8752cc84e411b4'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'thy life under thy mother', 'type': 'extractive', 'score': 0.024128902703523636, 'context': 'Termination of activity, cessation from movement and opinion, and in a sense their death, is no evil. Turn thy thoughts now to the consideration of thy life, thy life as a child, as a youth, thy manhood, thy old age, for in these also every change was a death. Is this anything to fear? Turn thy thoughts now to thy life under thy grandfather, then to thy life under thy mother, then to thy life under thy father; and as thou findest many other differences and changes and terminations, ask thyself, Is this anything to fear? In like manner, then, neither are the termination and cessation and change of thy whole life a thing to be afraid of.', 'offsets_in_document': [{'start': 352, 'end': 377}], 'offsets_in_context': [{'start': 352, 'end': 377}], 'document_ids': ['c18a0187633896c0b4275e044bf742ee'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'good teachers', 'type': 'extractive', 'score': 0.011039121076464653, 'context': 'From my great-grandfather, not to have frequented public schools, and to have had good teachers at home, and to know that on such things a man should spend liberally.', 'offsets_in_document': [{'start': 82, 'end': 95}], 'offsets_in_context': [{'start': 82, 'end': 95}], 'document_ids': ['366d76bbaee1f45066c8986efe265a0b'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'self-government', 'type': 'extractive', 'score': 0.004011611919850111, 'context': 'From Maximus I learned self-government, and not to be led aside by anything; and cheerfulness in all circumstances, as well as in illness; and a just admixture in the moral character of sweetness and dignity, and to do what was set before me without complaining. I observed that everybody believed that he thought as he spoke, and that in all that he did he never had any bad intention; and he never showed amazement and surprise, and was never in a hurry, and never put off doing a thing, nor was perplexed nor dejected, nor did he ever laugh to disguise his vexation, nor, on the other hand, was he ever passionate or suspicious. He was accustomed to do acts of beneficence, and was ready to forgive, and was free from all falsehood; and he presented the appearance of a man who could not be diverted from right rather than of a man who had been improved. I observed, too, that no man could ever think that he was despised by Maximus, or ever venture to think himself a better man. He had also the art of being humorous in an agreeable way.', 'offsets_in_document': [{'start': 23, 'end': 38}], 'offsets_in_context': [{'start': 23, 'end': 38}], 'document_ids': ['798a2fd2d075fe24606a777f348aca3d'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'Teach them then or bear', 'type': 'extractive', 'score': 0.003927578218281269, 'context': 'Men exist for the sake of one another. Teach them then or bear with them.', 'offsets_in_document': [{'start': 39, 'end': 62}], 'offsets_in_context': [{'start': 39, 'end': 62}], 'document_ids': ['2150cbb98d0d573101968513d7fb4af'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'to be themselves praised by posterity, by those whom they have never seen or ever will see, this they set much value on. But this is very much the same as if thou shouldst be grieved because those who have lived before thee did not praise thee', 'type': 'extractive', 'score': 0.00030686991522088647, 'context': 'How strangely men act. They will not praise those who are living at the same time and living with themselves; but to be themselves praised by posterity, by those whom they have never seen or ever will see, this they set much value on. But this is very much the same as if thou shouldst be grieved because those who have lived before thee did not praise thee.', 'offsets_in_document': [{'start': 114, 'end': 357}], 'offsets_in_context': [{'start': 114, 'end': 357}], 'document_ids': ['3ea45896a0ea4ae48663003a48029424'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'philosophy', 'type': 'extractive', 'score': 0.00030287448316812515, 'context': 'Epicurus says, In my sickness my conversation was not about my bodily sufferings, nor, says he, did I talk on such subjects to those who visited me; but I continued to discourse on the nature of things as before, keeping to this main point, how the mind, while participating in such movements as go on in the poor flesh, shall be free from perturbations and maintain its proper good. Nor did I, he says, give the physicians an opportunity of putting on solemn looks, as if they were doing something great, but my life went on well and happily. Do, then, the same that he did both in sickness, if thou art sick, and in any other circumstances; for never to desert philosophy in any events that may befall us, nor to hold trifling talk either with an ignorant man or with one unacquainted with nature, is a principle of all schools of philosophy; but to be intent only on that which thou art now doing and on the instrument by which thou doest it.', 'offsets_in_document': [{'start': 663, 'end': 673}], 'offsets_in_context': [{'start': 663, 'end': 673}], 'document_ids': ['b88b5ac821f7c6a2ae636cce2d882f88'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'nature and profitable', 'type': 'extractive', 'score': 0.00016086427785921842, 'context': 'How cruel it is not to allow men to strive after the things which appear to them to be suitable to their nature and profitable! And yet in a manner thou dost not allow them to do this, when thou art vexed because they do wrong. For they are certainly moved towards things because they suppose them to be suitable to their nature and profitable to them.- But it is not so.- Teach them then, and show them without being angry.', 'offsets_in_document': [{'start': 105, 'end': 126}], 'offsets_in_context': [{'start': 105, 'end': 126}], 'document_ids': ['8149dd43100c073a25aeec95588ac011'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'foul smell and blood in a bag', 'type': 'extractive', 'score': 0.00011568663467187434, 'context': 'Does Panthea or Pergamus now sit by the tomb of Verus? Does Chaurias or Diotimus sit by the tomb of Hadrian? That would be ridiculous. Well, suppose they did sit there, would the dead be conscious of it? And if the dead were conscious, would they be pleased? And if they were pleased, would that make them immortal? Was it not in the order of destiny that these persons too should first become old women and old men and then die? What then would those do after these were dead? All this is foul smell and blood in a bag.', 'offsets_in_document': [{'start': 490, 'end': 519}], 'offsets_in_context': [{'start': 490, 'end': 519}], 'document_ids': ['dd42d5594f0dc8416537a3b3c1f1d61c'], 'meta': {'source': 'meditations'}}>,\n",
              "  <Answer {'answer': 'fig-tree to have juice', 'type': 'extractive', 'score': 5.6748202041490003e-05, 'context': 'It is natural that these things should be done by such persons, it is a matter of necessity; and if a man will not have it so, he will not allow the fig-tree to have juice. But by all means bear this in mind, that within a very short time both thou and he will be dead; and soon not even your names will be left behind.', 'offsets_in_document': [{'start': 149, 'end': 171}], 'offsets_in_context': [{'start': 149, 'end': 171}], 'document_ids': ['c8f99a89c5399c2ad0a001747f8db0d1'], 'meta': {'source': 'meditations'}}>],\n",
              " 'documents': [<Document: {'content': 'Men exist for the sake of one another. Teach them then or bear with them.', 'content_type': 'text', 'score': 0.7322221779074343, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '2150cbb98d0d573101968513d7fb4af'}>,\n",
              "  <Document: {'content': 'From my grandfather Verus I learned good morals and the government of my temper.', 'content_type': 'text', 'score': 0.7208915763758528, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': 'ecc466cb76ed8dedab8752cc84e411b4'}>,\n",
              "  <Document: {'content': 'From my great-grandfather, not to have frequented public schools, and to have had good teachers at home, and to know that on such things a man should spend liberally.', 'content_type': 'text', 'score': 0.7011058050766118, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '366d76bbaee1f45066c8986efe265a0b'}>,\n",
              "  <Document: {'content': 'It is natural that these things should be done by such persons, it is a matter of necessity; and if a man will not have it so, he will not allow the fig-tree to have juice. But by all means bear this in mind, that within a very short time both thou and he will be dead; and soon not even your names will be left behind.', 'content_type': 'text', 'score': 0.6908060010236104, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': 'c8f99a89c5399c2ad0a001747f8db0d1'}>,\n",
              "  <Document: {'content': 'How cruel it is not to allow men to strive after the things which appear to them to be suitable to their nature and profitable! And yet in a manner thou dost not allow them to do this, when thou art vexed because they do wrong. For they are certainly moved towards things because they suppose them to be suitable to their nature and profitable to them.- But it is not so.- Teach them then, and show them without being angry.', 'content_type': 'text', 'score': 0.6661904567030548, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '8149dd43100c073a25aeec95588ac011'}>,\n",
              "  <Document: {'content': 'Epicurus says, In my sickness my conversation was not about my bodily sufferings, nor, says he, did I talk on such subjects to those who visited me; but I continued to discourse on the nature of things as before, keeping to this main point, how the mind, while participating in such movements as go on in the poor flesh, shall be free from perturbations and maintain its proper good. Nor did I, he says, give the physicians an opportunity of putting on solemn looks, as if they were doing something great, but my life went on well and happily. Do, then, the same that he did both in sickness, if thou art sick, and in any other circumstances; for never to desert philosophy in any events that may befall us, nor to hold trifling talk either with an ignorant man or with one unacquainted with nature, is a principle of all schools of philosophy; but to be intent only on that which thou art now doing and on the instrument by which thou doest it.', 'content_type': 'text', 'score': 0.6510449397543675, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': 'b88b5ac821f7c6a2ae636cce2d882f88'}>,\n",
              "  <Document: {'content': 'From Maximus I learned self-government, and not to be led aside by anything; and cheerfulness in all circumstances, as well as in illness; and a just admixture in the moral character of sweetness and dignity, and to do what was set before me without complaining. I observed that everybody believed that he thought as he spoke, and that in all that he did he never had any bad intention; and he never showed amazement and surprise, and was never in a hurry, and never put off doing a thing, nor was perplexed nor dejected, nor did he ever laugh to disguise his vexation, nor, on the other hand, was he ever passionate or suspicious. He was accustomed to do acts of beneficence, and was ready to forgive, and was free from all falsehood; and he presented the appearance of a man who could not be diverted from right rather than of a man who had been improved. I observed, too, that no man could ever think that he was despised by Maximus, or ever venture to think himself a better man. He had also the art of being humorous in an agreeable way.', 'content_type': 'text', 'score': 0.6425392669963337, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '798a2fd2d075fe24606a777f348aca3d'}>,\n",
              "  <Document: {'content': 'Does Panthea or Pergamus now sit by the tomb of Verus? Does Chaurias or Diotimus sit by the tomb of Hadrian? That would be ridiculous. Well, suppose they did sit there, would the dead be conscious of it? And if the dead were conscious, would they be pleased? And if they were pleased, would that make them immortal? Was it not in the order of destiny that these persons too should first become old women and old men and then die? What then would those do after these were dead? All this is foul smell and blood in a bag.', 'content_type': 'text', 'score': 0.6408262431774636, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': 'dd42d5594f0dc8416537a3b3c1f1d61c'}>,\n",
              "  <Document: {'content': 'Termination of activity, cessation from movement and opinion, and in a sense their death, is no evil. Turn thy thoughts now to the consideration of thy life, thy life as a child, as a youth, thy manhood, thy old age, for in these also every change was a death. Is this anything to fear? Turn thy thoughts now to thy life under thy grandfather, then to thy life under thy mother, then to thy life under thy father; and as thou findest many other differences and changes and terminations, ask thyself, Is this anything to fear? In like manner, then, neither are the termination and cessation and change of thy whole life a thing to be afraid of.', 'content_type': 'text', 'score': 0.6366416815738632, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': 'c18a0187633896c0b4275e044bf742ee'}>,\n",
              "  <Document: {'content': 'How strangely men act. They will not praise those who are living at the same time and living with themselves; but to be themselves praised by posterity, by those whom they have never seen or ever will see, this they set much value on. But this is very much the same as if thou shouldst be grieved because those who have lived before thee did not praise thee.', 'content_type': 'text', 'score': 0.6277992522181182, 'meta': {'source': 'meditations'}, 'id_hash_keys': ['content'], 'embedding': None, 'id': '3ea45896a0ea4ae48663003a48029424'}>],\n",
              " 'root_node': 'Query',\n",
              " 'params': {},\n",
              " 'node_id': 'Reader'}"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "OnpcI0IlqmBx"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}