Bio_SciBERT_Pytorch_FulltextQA.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Bio/SciBERT_Pytorch_FulltextQA.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "2011ff90bd1d4111b02e5bed6dd5f4a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_e7660af5c3ea4fa7b3006e2a7583e9da",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_9115a8eba65d493caa6b03daab93d145",
              "IPY_MODEL_1b765c40ee1e4df7bb9f2876251cba2d"
            ]
          }
        },
        "e7660af5c3ea4fa7b3006e2a7583e9da": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "9115a8eba65d493caa6b03daab93d145": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_2a7d060a810541549da19a6c723ca9eb",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 222296,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 222296,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_5f55edf7c02b446a9cbceed5eeca81a1"
          }
        },
        "1b765c40ee1e4df7bb9f2876251cba2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_f6f126b09b21479e902c42234c1bdb0e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 222k/222k [00:00&lt;00:00, 1.43MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_e0b0908880d84f3b8be1f89f21844961"
          }
        },
        "2a7d060a810541549da19a6c723ca9eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "5f55edf7c02b446a9cbceed5eeca81a1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "f6f126b09b21479e902c42234c1bdb0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "e0b0908880d84f3b8be1f89f21844961": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "5a2ba2502ab640b0b785543c5056aa26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_bbcb173260ed4b62ba9405dabb5e1679",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_0b859347e7b24f31babe6790e8a5f3da",
              "IPY_MODEL_11fce564b4c9429eb25c936303cfcd2f"
            ]
          }
        },
        "bbcb173260ed4b62ba9405dabb5e1679": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "0b859347e7b24f31babe6790e8a5f3da": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_506894b81c664c32a962b4e69f261416",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 442301670,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442301670,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b4fff930b2084ef2a4c66050d940559e"
          }
        },
        "11fce564b4c9429eb25c936303cfcd2f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8e01c74741fd444e8b13c0d5bcfb3d0e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442M/442M [00:09&lt;00:00, 45.5MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_25977feffef648eab4cb9f096f77424b"
          }
        },
        "506894b81c664c32a962b4e69f261416": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b4fff930b2084ef2a4c66050d940559e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8e01c74741fd444e8b13c0d5bcfb3d0e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "25977feffef648eab4cb9f096f77424b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z_J_KWRSSJpO",
        "outputId": "fb207fb3-2839-4d18-bb59-6406117726f6"
      },
      "source": [
        "!pip install torch==1.8.0+cpu torchvision==0.9.0+cpu torchaudio===0.8.0 -f https://download.pytorch.org/whl/torch_stable.html"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n",
            "Collecting torch==1.8.0+cpu\n",
            "\u001b[?25l  Downloading https://download.pytorch.org/whl/cpu/torch-1.8.0%2Bcpu-cp37-cp37m-linux_x86_64.whl (169.1MB)\n",
            "\u001b[K     |████████████████████████████████| 169.1MB 74kB/s \n",
            "\u001b[?25hCollecting torchvision==0.9.0+cpu\n",
            "\u001b[?25l  Downloading https://download.pytorch.org/whl/cpu/torchvision-0.9.0%2Bcpu-cp37-cp37m-linux_x86_64.whl (13.3MB)\n",
            "\u001b[K     |████████████████████████████████| 13.4MB 273kB/s \n",
            "\u001b[?25hCollecting torchaudio===0.8.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/18/9a/4e2e6dbde627ffb8a6d1d4ebc4683edecad1c08099969f1d7760d92175ff/torchaudio-0.8.0-cp37-cp37m-manylinux1_x86_64.whl (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 38.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch==1.8.0+cpu) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch==1.8.0+cpu) (3.7.4.3)\n",
            "Requirement already satisfied: pillow>=4.1.1 in /usr/local/lib/python3.7/dist-packages (from torchvision==0.9.0+cpu) (7.1.2)\n",
            "\u001b[31mERROR: torchtext 0.9.1 has requirement torch==1.8.1, but you'll have torch 1.8.0+cpu which is incompatible.\u001b[0m\n",
            "Installing collected packages: torch, torchvision, torchaudio\n",
            "  Found existing installation: torch 1.8.1+cu101\n",
            "    Uninstalling torch-1.8.1+cu101:\n",
            "      Successfully uninstalled torch-1.8.1+cu101\n",
            "  Found existing installation: torchvision 0.9.1+cu101\n",
            "    Uninstalling torchvision-0.9.1+cu101:\n",
            "      Successfully uninstalled torchvision-0.9.1+cu101\n",
            "Successfully installed torch-1.8.0+cpu torchaudio-0.8.0 torchvision-0.9.0+cpu\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_XI6Dxb1S1uB",
        "outputId": "1b78de30-f060-4e23-c8af-b965dbebc927"
      },
      "source": [
        "!pip install transformers==3.0.0"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting transformers==3.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9c/35/1c3f6e62d81f5f0daff1384e6d5e6c5758682a8357ebc765ece2b9def62b/transformers-3.0.0-py3-none-any.whl (754kB)\n",
            "\r\u001b[K     |▍                               | 10kB 21.7MB/s eta 0:00:01\r\u001b[K     |▉                               | 20kB 26.9MB/s eta 0:00:01\r\u001b[K     |█▎                              | 30kB 32.4MB/s eta 0:00:01\r\u001b[K     |█▊                              | 40kB 28.8MB/s eta 0:00:01\r\u001b[K     |██▏                             | 51kB 28.1MB/s eta 0:00:01\r\u001b[K     |██▋                             | 61kB 30.3MB/s eta 0:00:01\r\u001b[K     |███                             | 71kB 26.3MB/s eta 0:00:01\r\u001b[K     |███▌                            | 81kB 27.3MB/s eta 0:00:01\r\u001b[K     |████                            | 92kB 28.7MB/s eta 0:00:01\r\u001b[K     |████▍                           | 102kB 27.8MB/s eta 0:00:01\r\u001b[K     |████▊                           | 112kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 122kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████▋                          | 133kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████                          | 143kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 153kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████                         | 163kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 174kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████▉                        | 184kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 194kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 204kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 215kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 225kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████                      | 235kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 245kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 256kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 266kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 276kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████▏                   | 286kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 296kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 307kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 317kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 327kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████▎                 | 337kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 348kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 358kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████▋                | 368kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████                | 378kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████▌               | 389kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 399kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 409kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 419kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 430kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████▊             | 440kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 450kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 460kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 471kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 481kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████▉           | 491kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 501kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 512kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▏         | 522kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 532kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 542kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 552kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 563kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 573kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 583kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 593kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 604kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 614kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▌     | 624kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 634kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▍    | 645kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 655kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 665kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 675kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 686kB 27.8MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 696kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 706kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 716kB 27.8MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▉ | 727kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 737kB 27.8MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▊| 747kB 27.8MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 757kB 27.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.0) (1.19.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.0) (20.9)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 42.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.0) (2.23.0)\n",
            "Collecting tokenizers==0.8.0-rc4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f7/82/0e82a95bd9db2b32569500cc1bb47aa7c4e0f57aa5e35cceba414096917b/tokenizers-0.8.0rc4-cp37-cp37m-manylinux1_x86_64.whl (3.0MB)\n",
            "\u001b[K     |████████████████████████████████| 3.0MB 48.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.0) (4.41.1)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 31.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.0) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers==3.0.0) (2019.12.20)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers==3.0.0) (2.4.7)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.0) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.0) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.0) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers==3.0.0) (2020.12.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.0.0) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.0.0) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers==3.0.0) (1.0.1)\n",
            "Installing collected packages: sentencepiece, tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.45 sentencepiece-0.1.95 tokenizers-0.8.0rc4 transformers-3.0.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xVeUjOlRTQvA",
        "outputId": "f0617484-312e-492b-8c28-6eba58e0d35e"
      },
      "source": [
        "!pip install ktrain"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting ktrain\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/67/31cab9d7c0e23333aebc28b082659c1528f9ab7e22d00e7237efe4fc14f6/ktrain-0.26.2.tar.gz (25.3MB)\n",
            "\u001b[K     |████████████████████████████████| 25.3MB 121kB/s \n",
            "\u001b[?25hCollecting scikit-learn==0.23.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f4/cb/64623369f348e9bfb29ff898a57ac7c91ed4921f228e9726546614d63ccb/scikit_learn-0.23.2-cp37-cp37m-manylinux1_x86_64.whl (6.8MB)\n",
            "\u001b[K     |████████████████████████████████| 6.8MB 24.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from ktrain) (3.2.2)\n",
            "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.1.5)\n",
            "Requirement already satisfied: fastprogress>=0.1.21 in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.0.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from ktrain) (2.23.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.0.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from ktrain) (20.9)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.7/dist-packages (from ktrain) (5.5.0)\n",
            "Collecting langdetect\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz (981kB)\n",
            "\u001b[K     |████████████████████████████████| 983kB 42.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: jieba in /usr/local/lib/python3.7/dist-packages (from ktrain) (0.42.1)\n",
            "Collecting cchardet\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/80/72/a4fba7559978de00cf44081c548c5d294bf00ac7dcda2db405d2baa8c67a/cchardet-2.1.7-cp37-cp37m-manylinux2010_x86_64.whl (263kB)\n",
            "\u001b[K     |████████████████████████████████| 266kB 30.2MB/s \n",
            "\u001b[?25hCollecting syntok\n",
            "  Downloading https://files.pythonhosted.org/packages/8c/76/a49e73a04b3e3a14ce232e8e28a1587f8108baa665644fe8c40e307e792e/syntok-1.3.1.tar.gz\n",
            "Collecting seqeval==0.0.19\n",
            "  Downloading https://files.pythonhosted.org/packages/93/e5/b7705156a77f742cfe4fc6f22d0c71591edb2d243328dff2f8fc0f933ab6/seqeval-0.0.19.tar.gz\n",
            "Collecting transformers<=4.3.3,>=4.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/54/5ca07ec9569d2f232f3166de5457b63943882f7950ddfcc887732fc7fb23/transformers-4.3.3-py3-none-any.whl (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 24.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: sentencepiece in /usr/local/lib/python3.7/dist-packages (from ktrain) (0.1.95)\n",
            "Collecting keras_bert>=0.86.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e2/7f/95fabd29f4502924fa3f09ff6538c5a7d290dfef2c2fe076d3d1a16e08f0/keras-bert-0.86.0.tar.gz\n",
            "Requirement already satisfied: networkx>=2.3 in /usr/local/lib/python3.7/dist-packages (from ktrain) (2.5.1)\n",
            "Collecting whoosh\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ba/19/24d0f1f454a2c1eb689ca28d2f178db81e5024f42d82729a4ff6771155cf/Whoosh-2.7.4-py2.py3-none-any.whl (468kB)\n",
            "\u001b[K     |████████████████████████████████| 471kB 52.8MB/s \n",
            "\u001b[?25hCollecting threadpoolctl>=2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f7/12/ec3f2e203afa394a149911729357aa48affc59c20e2c1c8297a60f33f133/threadpoolctl-2.1.0-py3-none-any.whl\n",
            "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.2->ktrain) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.2->ktrain) (1.19.5)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (1.3.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (2.8.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.1->ktrain) (2018.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (1.24.3)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (2.10)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (3.0.4)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (4.8.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (4.4.2)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (0.7.5)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (0.8.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (1.0.18)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (2.6.1)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (56.1.0)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (5.0.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from langdetect->ktrain) (1.15.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from syntok->ktrain) (2019.12.20)\n",
            "Requirement already satisfied: Keras>=2.2.4 in /usr/local/lib/python3.7/dist-packages (from seqeval==0.0.19->ktrain) (2.4.3)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (0.0.45)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 33.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (4.0.1)\n",
            "Collecting keras-transformer>=0.38.0\n",
            "  Downloading https://files.pythonhosted.org/packages/89/6c/d6f0c164f4cc16fbc0d0fea85f5526e87a7d2df7b077809e422a7e626150/keras-transformer-0.38.0.tar.gz\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect; sys_platform != \"win32\"->ipython->ktrain) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->ktrain) (0.2.5)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from traitlets>=4.2->ipython->ktrain) (0.2.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from Keras>=2.2.4->seqeval==0.0.19->ktrain) (3.13)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from Keras>=2.2.4->seqeval==0.0.19->ktrain) (2.10.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<=4.3.3,>=4.0.0->ktrain) (7.1.2)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<=4.3.3,>=4.0.0->ktrain) (3.4.1)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<=4.3.3,>=4.0.0->ktrain) (3.7.4.3)\n",
            "Collecting keras-pos-embd>=0.11.0\n",
            "  Downloading https://files.pythonhosted.org/packages/09/70/b63ed8fc660da2bb6ae29b9895401c628da5740c048c190b5d7107cadd02/keras-pos-embd-0.11.0.tar.gz\n",
            "Collecting keras-multi-head>=0.27.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e6/32/45adf2549450aca7867deccfa04af80a0ab1ca139af44b16bc669e0e09cd/keras-multi-head-0.27.0.tar.gz\n",
            "Collecting keras-layer-normalization>=0.14.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a4/0e/d1078df0494bac9ce1a67954e5380b6e7569668f0f3b50a9531c62c1fc4a/keras-layer-normalization-0.14.0.tar.gz\n",
            "Collecting keras-position-wise-feed-forward>=0.6.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e3/59/f0faa1037c033059e7e9e7758e6c23b4d1c0772cd48de14c4b6fd4033ad5/keras-position-wise-feed-forward-0.6.0.tar.gz\n",
            "Collecting keras-embed-sim>=0.8.0\n",
            "  Downloading https://files.pythonhosted.org/packages/57/ef/61a1e39082c9e1834a2d09261d4a0b69f7c818b359216d4e1912b20b1c86/keras-embed-sim-0.8.0.tar.gz\n",
            "Collecting keras-self-attention==0.46.0\n",
            "  Downloading https://files.pythonhosted.org/packages/15/6b/c804924a056955fa1f3ff767945187103cfc851ba9bd0fc5a6c6bc18e2eb/keras-self-attention-0.46.0.tar.gz\n",
            "Building wheels for collected packages: ktrain, langdetect, syntok, seqeval, keras-bert, keras-transformer, keras-pos-embd, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-self-attention\n",
            "  Building wheel for ktrain (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ktrain: filename=ktrain-0.26.2-cp37-none-any.whl size=25277794 sha256=b652183e3a9d26650a611bfb52b04414584e4b16bf9b31c9f63a48718f74d933\n",
            "  Stored in directory: /root/.cache/pip/wheels/67/2e/f1/c72afa08df8b2d984b910dea228902ce81dae4511afe9fafd2\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-cp37-none-any.whl size=993223 sha256=143f8966f8e5e03c403c3aa5e81bc96429814c2f5474f01b76fddb6279de2ece\n",
            "  Stored in directory: /root/.cache/pip/wheels/7e/18/13/038c34057808931c7ddc6c92d3aa015cf1a498df5a70268996\n",
            "  Building wheel for syntok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for syntok: filename=syntok-1.3.1-cp37-none-any.whl size=20919 sha256=0920a50a22d0d9b882eebf5d8316d26bf15e87df2e112e407aa7105ff37c8598\n",
            "  Stored in directory: /root/.cache/pip/wheels/51/c6/a4/be1920586c49469846bcd2888200bdecfe109ec421dab9be2d\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-0.0.19-cp37-none-any.whl size=9919 sha256=224a955df356e648ebfab8c8ebf5370a92a1225661938c4a930006c89e0d5514\n",
            "  Stored in directory: /root/.cache/pip/wheels/8d/1f/bf/1198beceed805a2099060975f6281d1b01046dd279e19c97be\n",
            "  Building wheel for keras-bert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-bert: filename=keras_bert-0.86.0-cp37-none-any.whl size=34144 sha256=8712c1f7b85054a47ec6074ef8da9d7b3a9b38039f5481a1c1c795760590ab81\n",
            "  Stored in directory: /root/.cache/pip/wheels/66/f0/b1/748128b58562fc9e31b907bb5e2ab6a35eb37695e83911236b\n",
            "  Building wheel for keras-transformer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-transformer: filename=keras_transformer-0.38.0-cp37-none-any.whl size=12942 sha256=d89dfa5470d299513339bc97a2688697d3126ccb8be52492a02a07265033dd24\n",
            "  Stored in directory: /root/.cache/pip/wheels/e5/fb/3a/37b2b9326c799aa010ae46a04ddb04f320d8c77c0b7e837f4e\n",
            "  Building wheel for keras-pos-embd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-pos-embd: filename=keras_pos_embd-0.11.0-cp37-none-any.whl size=7554 sha256=fbfe576c85fa7df66b1dd43f00b4145d84e2ede77e6ce6550b7733bd80596e4d\n",
            "  Stored in directory: /root/.cache/pip/wheels/5b/a1/a0/ce6b1d49ba1a9a76f592e70cf297b05c96bc9f418146761032\n",
            "  Building wheel for keras-multi-head (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-multi-head: filename=keras_multi_head-0.27.0-cp37-none-any.whl size=15611 sha256=3bf5720d337514994418e93b5c300b99ce17188cb1306e332d8539e2f0fb0e65\n",
            "  Stored in directory: /root/.cache/pip/wheels/b5/b4/49/0a0c27dcb93c13af02fea254ff51d1a43a924dd4e5b7a7164d\n",
            "  Building wheel for keras-layer-normalization (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-layer-normalization: filename=keras_layer_normalization-0.14.0-cp37-none-any.whl size=5269 sha256=56fae16ac090ca6c657674c757c6ca31ccee9ab957c2b1538d877138e627f02f\n",
            "  Stored in directory: /root/.cache/pip/wheels/54/80/22/a638a7d406fd155e507aa33d703e3fa2612b9eb7bb4f4fe667\n",
            "  Building wheel for keras-position-wise-feed-forward (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-position-wise-feed-forward: filename=keras_position_wise_feed_forward-0.6.0-cp37-none-any.whl size=5623 sha256=5d5966ee7f1338e6103991c0b3a224b36843d925255b8fdd84bb4624dcf46c3c\n",
            "  Stored in directory: /root/.cache/pip/wheels/39/e2/e2/3514fef126a00574b13bc0b9e23891800158df3a3c19c96e3b\n",
            "  Building wheel for keras-embed-sim (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-embed-sim: filename=keras_embed_sim-0.8.0-cp37-none-any.whl size=4558 sha256=91d552dcdc621d943f35137e7e96b8c93fa6b92c24616e60e402f39b18f5ee69\n",
            "  Stored in directory: /root/.cache/pip/wheels/49/45/8b/c111f6cc8bec253e984677de73a6f4f5d2f1649f42aac191c8\n",
            "  Building wheel for keras-self-attention (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-self-attention: filename=keras_self_attention-0.46.0-cp37-none-any.whl size=17278 sha256=539f5b8dd3db3a2fc286d076c83a87319f6dd68ceb9eb69a5a1ad62b4211eea3\n",
            "  Stored in directory: /root/.cache/pip/wheels/d2/2e/80/fec4c05eb23c8e13b790e26d207d6e0ffe8013fad8c6bdd4d2\n",
            "Successfully built ktrain langdetect syntok seqeval keras-bert keras-transformer keras-pos-embd keras-multi-head keras-layer-normalization keras-position-wise-feed-forward keras-embed-sim keras-self-attention\n",
            "Installing collected packages: threadpoolctl, scikit-learn, langdetect, cchardet, syntok, seqeval, tokenizers, transformers, keras-pos-embd, keras-self-attention, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-transformer, keras-bert, whoosh, ktrain\n",
            "  Found existing installation: scikit-learn 0.22.2.post1\n",
            "    Uninstalling scikit-learn-0.22.2.post1:\n",
            "      Successfully uninstalled scikit-learn-0.22.2.post1\n",
            "  Found existing installation: tokenizers 0.8.0rc4\n",
            "    Uninstalling tokenizers-0.8.0rc4:\n",
            "      Successfully uninstalled tokenizers-0.8.0rc4\n",
            "  Found existing installation: transformers 3.0.0\n",
            "    Uninstalling transformers-3.0.0:\n",
            "      Successfully uninstalled transformers-3.0.0\n",
            "Successfully installed cchardet-2.1.7 keras-bert-0.86.0 keras-embed-sim-0.8.0 keras-layer-normalization-0.14.0 keras-multi-head-0.27.0 keras-pos-embd-0.11.0 keras-position-wise-feed-forward-0.6.0 keras-self-attention-0.46.0 keras-transformer-0.38.0 ktrain-0.26.2 langdetect-1.0.9 scikit-learn-0.23.2 seqeval-0.0.19 syntok-1.3.1 threadpoolctl-2.1.0 tokenizers-0.10.2 transformers-4.3.3 whoosh-2.7.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9MiqM6goTX7w"
      },
      "source": [
        "import ktrain\n",
        "import pandas as pd"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HvsspQieTe-P"
      },
      "source": [
        "import pickle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "x2yJelf2T0XY"
      },
      "source": [
        "from ktrain import text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TXlb3Cv3LQIS",
        "outputId": "ab4357cd-bbf4-46f2-d03e-a5a9d6e03dbe"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oUdMiSahTuF_"
      },
      "source": [
        "#X_train = pd.read_pickle('/content/drive/MyDrive/X_train.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Bo1ogbCBT7SI"
      },
      "source": [
        "#X_test = pd.read_pickle('/content/drive/MyDrive/X_test.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Zn1PkLMcT_eF"
      },
      "source": [
        "#y_train = pd.read_pickle('/content/drive/MyDrive/y_train.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Pf7BJ-iOUCmK"
      },
      "source": [
        "#y_test = pd.read_pickle('/content/drive/MyDrive/y_test.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lfhH9G2a6OIF"
      },
      "source": [
        "#y_train"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Inqi-jUhVCSj"
      },
      "source": [
        "df = pd.read_pickle('save_path/preproQA.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "11pu0Wh-MvDj"
      },
      "source": [
        "df = df[['abs_titl_qa','FuL']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "AELlIcEzMzbt",
        "outputId": "68761284-08c2-4066-b747-cf81e2c1f986"
      },
      "source": [
        "df.FuL.value_counts()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.0    45\n",
              "1.0    42\n",
              "Name: FuL, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "tXKJv09YWyvZ"
      },
      "source": [
        "df_valid = df.sample(frac = 0.2, random_state = 42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LMTZASRtW1OY"
      },
      "source": [
        "df_train = df.drop(df_valid.index)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "OUnCSRvSbRTG"
      },
      "source": [
        "(X_train, y_train) = (df_train.abs_titl_qa, df_train.FuL)\n",
        "(X_test, y_test) = (df_valid.abs_titl_qa, df_valid.FuL)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EgteiYqobbnu"
      },
      "source": [
        "X_train = X_train.tolist()\n",
        "X_test = X_test.tolist()\n",
        "y_test = y_test.tolist()\n",
        "y_train = y_train.tolist()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "O4VTOVpRUfDZ"
      },
      "source": [
        "MODEL_NAME = 'allenai/scibert_scivocab_cased'  #name of pretrained bert models from hugging face"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "eBUyRrcxUvNZ"
      },
      "source": [
        "t = text.Transformer(MODEL_NAME, maxlen=512, class_names=[0, 1])"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 245,
          "referenced_widgets": [
            "2011ff90bd1d4111b02e5bed6dd5f4a1",
            "e7660af5c3ea4fa7b3006e2a7583e9da",
            "9115a8eba65d493caa6b03daab93d145",
            "1b765c40ee1e4df7bb9f2876251cba2d",
            "2a7d060a810541549da19a6c723ca9eb",
            "5f55edf7c02b446a9cbceed5eeca81a1",
            "f6f126b09b21479e902c42234c1bdb0e",
            "e0b0908880d84f3b8be1f89f21844961"
          ]
        },
        "id": "0GGrMqtRU44H",
        "outputId": "7ea5b16c-c597-40be-ca8a-88119115bc73"
      },
      "source": [
        "trn = t.preprocess_train(X_train, y_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 186\n",
            "\t95percentile : 247\n",
            "\t99percentile : 357\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ktrain/utils.py:605: UserWarning: class_names implies classification but targets array contains float(s) instead of integers or strings\n",
            "  warnings.warn('class_names implies classification but targets array contains float(s) instead of integers or strings')\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2011ff90bd1d4111b02e5bed6dd5f4a1",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=222296.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 178
        },
        "id": "E-rYxYueU9QX",
        "outputId": "f3eed807-c82e-485d-aa8d-9492f6a8d9c1"
      },
      "source": [
        "val = t.preprocess_test(X_test, y_test)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 191\n",
            "\t95percentile : 239\n",
            "\t99percentile : 248\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ktrain/utils.py:605: UserWarning: class_names implies classification but targets array contains float(s) instead of integers or strings\n",
            "  warnings.warn('class_names implies classification but targets array contains float(s) instead of integers or strings')\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 84,
          "referenced_widgets": [
            "5a2ba2502ab640b0b785543c5056aa26",
            "bbcb173260ed4b62ba9405dabb5e1679",
            "0b859347e7b24f31babe6790e8a5f3da",
            "11fce564b4c9429eb25c936303cfcd2f",
            "506894b81c664c32a962b4e69f261416",
            "b4fff930b2084ef2a4c66050d940559e",
            "8e01c74741fd444e8b13c0d5bcfb3d0e",
            "25977feffef648eab4cb9f096f77424b"
          ]
        },
        "id": "aeGsOOJYW3Z5",
        "outputId": "7509fca7-3399-4067-f321-3f393047b1a1"
      },
      "source": [
        "model = t.get_classifier()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "404 Client Error: Not Found for url: https://huggingface.co/allenai/scibert_scivocab_cased/resolve/main/tf_model.h5\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5a2ba2502ab640b0b785543c5056aa26",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442301670.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "rZZRLGbPXpJO"
      },
      "source": [
        "learner = ktrain.get_learner(model, train_data=trn, val_data=val, batch_size=6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Xb6T-MlkXttP",
        "outputId": "3f001d8e-9b43-4e8b-81b6-bbd8f1989692"
      },
      "source": [
        "learner.fit_onecycle(2e-5, 10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "12/12 [==============================] - 20s 822ms/step - loss: 0.6524 - accuracy: 0.6310 - val_loss: 0.7028 - val_accuracy: 0.5294\n",
            "Epoch 2/10\n",
            "12/12 [==============================] - 9s 729ms/step - loss: 0.7470 - accuracy: 0.4764 - val_loss: 0.6833 - val_accuracy: 0.5882\n",
            "Epoch 3/10\n",
            "12/12 [==============================] - 9s 736ms/step - loss: 0.6832 - accuracy: 0.6344 - val_loss: 0.7003 - val_accuracy: 0.4706\n",
            "Epoch 4/10\n",
            "12/12 [==============================] - 9s 745ms/step - loss: 0.7099 - accuracy: 0.4082 - val_loss: 0.6749 - val_accuracy: 0.5882\n",
            "Epoch 5/10\n",
            "12/12 [==============================] - 9s 750ms/step - loss: 0.7670 - accuracy: 0.3281 - val_loss: 0.6865 - val_accuracy: 0.6471\n",
            "Epoch 6/10\n",
            "12/12 [==============================] - 9s 756ms/step - loss: 0.7162 - accuracy: 0.4513 - val_loss: 0.6816 - val_accuracy: 0.6471\n",
            "Epoch 7/10\n",
            "12/12 [==============================] - 9s 758ms/step - loss: 0.6598 - accuracy: 0.6130 - val_loss: 0.6807 - val_accuracy: 0.5882\n",
            "Epoch 8/10\n",
            "12/12 [==============================] - 9s 765ms/step - loss: 0.6888 - accuracy: 0.6394 - val_loss: 0.6873 - val_accuracy: 0.5294\n",
            "Epoch 9/10\n",
            "12/12 [==============================] - 9s 773ms/step - loss: 0.6520 - accuracy: 0.5908 - val_loss: 0.6710 - val_accuracy: 0.5294\n",
            "Epoch 10/10\n",
            "12/12 [==============================] - 9s 776ms/step - loss: 0.6132 - accuracy: 0.6856 - val_loss: 0.6539 - val_accuracy: 0.6471\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "<tensorflow.python.keras.callbacks.History at 0x7f5e408993d0>"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 32
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wwrMRGJEcEWX"
      },
      "source": [
        "from sklearn import metrics"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U2CCXIfRcIaH"
      },
      "source": [
        "predictor = ktrain.get_predictor(learner.model,preproc=t)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vM4OdfIxZxFa"
      },
      "source": [
        "pre_text=df_valid['abs_titl_qa']\n",
        "data=list(pre_text)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "S9N0MNmwkpU8"
      },
      "source": [
        "y_pred = predictor.predict(data, return_proba=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "M-DS120wktVa"
      },
      "source": [
        "df_valid.FuL = df_valid.FuL.astype('int')\n",
        "df_train.FuL = df_train.FuL.astype('int')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WeNkVtrGk2j2"
      },
      "source": [
        "y_test = df_valid['FuL']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SBFn-q8jk6tF",
        "outputId": "e7dcd688-3c3f-4175-9ac9-a8569a23880d"
      },
      "source": [
        "fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred[:,1])\n",
        "auc = metrics.auc(fpr, tpr)\n",
        "#plt.plot(fpr, tpr)\n",
        "auc"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6714285714285714"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    }
  ]
}