Learning_curve_distillBERTs_CorrectedAbs_addTil0.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.7.6"
    },
    "colab": {
      "name": "Learning_curve_distillBERTs_CorrectedAbs_addTil0.ipynb",
      "provenance": []
    },
    "accelerator": "GPU",
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "1f8ad812dbc84d6f8f718c898b166b40": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_979b2efae6b7483883c52f55723d37db",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_05d85b9bc1714fbaadc6ea04246a12e9",
              "IPY_MODEL_91c712547d984a40b53026d5e0bf2939"
            ]
          }
        },
        "979b2efae6b7483883c52f55723d37db": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "05d85b9bc1714fbaadc6ea04246a12e9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_4c7d0aee24e64fc7a9baa70a9468ba1f",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 442,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 442,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_925a34155673496fae1240c93f2290b6"
          }
        },
        "91c712547d984a40b53026d5e0bf2939": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_8eae09d2a2a2486e8ca9b229cc88cf41",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 442/442 [03:31&lt;00:00, 2.09B/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_b47a546b84da4b0e81b873058c1e8bb1"
          }
        },
        "4c7d0aee24e64fc7a9baa70a9468ba1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "925a34155673496fae1240c93f2290b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "8eae09d2a2a2486e8ca9b229cc88cf41": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "b47a546b84da4b0e81b873058c1e8bb1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "a1c8774ad45d421fa644536bf3e3dbff": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_4df33d4f07d447f5802be99b3d254bdd",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_908f1629c7714f9eaa00fd20f97f7c47",
              "IPY_MODEL_0ced40c126b648e694e95270935d2364"
            ]
          }
        },
        "4df33d4f07d447f5802be99b3d254bdd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "908f1629c7714f9eaa00fd20f97f7c47": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_435c97b291d948539c173c7adb359d63",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 231508,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 231508,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_69cca8431534429ba5d6d1511ebf036d"
          }
        },
        "0ced40c126b648e694e95270935d2364": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_776a98a9d85e47dd9187a733ec7d6425",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 232k/232k [00:01&lt;00:00, 186kB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_d590b6a4cdbe4191ac4c1a3725798344"
          }
        },
        "435c97b291d948539c173c7adb359d63": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "69cca8431534429ba5d6d1511ebf036d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "776a98a9d85e47dd9187a733ec7d6425": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "d590b6a4cdbe4191ac4c1a3725798344": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "e3b588ef2143444bb7a7900200abd5db": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_view_name": "HBoxView",
            "_dom_classes": [],
            "_model_name": "HBoxModel",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "box_style": "",
            "layout": "IPY_MODEL_9fd5c3faecc5455b84d06a5d8ca501a4",
            "_model_module": "@jupyter-widgets/controls",
            "children": [
              "IPY_MODEL_668f5c67be1841fbbac6bcc7785bde38",
              "IPY_MODEL_b4f040bc24db40a3ac47c816bca846eb"
            ]
          }
        },
        "9fd5c3faecc5455b84d06a5d8ca501a4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "668f5c67be1841fbbac6bcc7785bde38": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_view_name": "ProgressView",
            "style": "IPY_MODEL_5fc80178e3ad4b028c090ffa7b8626c5",
            "_dom_classes": [],
            "description": "Downloading: 100%",
            "_model_name": "FloatProgressModel",
            "bar_style": "success",
            "max": 466062,
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": 466062,
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "orientation": "horizontal",
            "min": 0,
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_dc9a6b1b8cbc43b3892d68ccf5c5b89a"
          }
        },
        "b4f040bc24db40a3ac47c816bca846eb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_view_name": "HTMLView",
            "style": "IPY_MODEL_2f5220e3e6954b3388f31c6ecd94229e",
            "_dom_classes": [],
            "description": "",
            "_model_name": "HTMLModel",
            "placeholder": "​",
            "_view_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "value": " 466k/466k [00:00&lt;00:00, 1.14MB/s]",
            "_view_count": null,
            "_view_module_version": "1.5.0",
            "description_tooltip": null,
            "_model_module": "@jupyter-widgets/controls",
            "layout": "IPY_MODEL_3ac586698fbf4bd78e5ac15bf1e72157"
          }
        },
        "5fc80178e3ad4b028c090ffa7b8626c5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "ProgressStyleModel",
            "description_width": "initial",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "bar_color": null,
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "dc9a6b1b8cbc43b3892d68ccf5c5b89a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        },
        "2f5220e3e6954b3388f31c6ecd94229e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_view_name": "StyleView",
            "_model_name": "DescriptionStyleModel",
            "description_width": "",
            "_view_module": "@jupyter-widgets/base",
            "_model_module_version": "1.5.0",
            "_view_count": null,
            "_view_module_version": "1.2.0",
            "_model_module": "@jupyter-widgets/controls"
          }
        },
        "3ac586698fbf4bd78e5ac15bf1e72157": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_view_name": "LayoutView",
            "grid_template_rows": null,
            "right": null,
            "justify_content": null,
            "_view_module": "@jupyter-widgets/base",
            "overflow": null,
            "_model_module_version": "1.2.0",
            "_view_count": null,
            "flex_flow": null,
            "width": null,
            "min_width": null,
            "border": null,
            "align_items": null,
            "bottom": null,
            "_model_module": "@jupyter-widgets/base",
            "top": null,
            "grid_column": null,
            "overflow_y": null,
            "overflow_x": null,
            "grid_auto_flow": null,
            "grid_area": null,
            "grid_template_columns": null,
            "flex": null,
            "_model_name": "LayoutModel",
            "justify_items": null,
            "grid_row": null,
            "max_height": null,
            "align_content": null,
            "visibility": null,
            "align_self": null,
            "height": null,
            "min_height": null,
            "padding": null,
            "grid_auto_rows": null,
            "grid_gap": null,
            "max_width": null,
            "order": null,
            "_view_module_version": "1.2.0",
            "grid_template_areas": null,
            "object_position": null,
            "object_fit": null,
            "grid_auto_columns": null,
            "margin": null,
            "display": null,
            "left": null
          }
        }
      }
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "HZ5_SJksVbR8"
      },
      "source": [
        "# Abstract-text-cleaning:leave numbers!\n",
        "# remove special characters and stop words (delete word indicate tense; creat a new stopword list)\n",
        "#(note: tense can help with the classification; protocol : future tense ; compeleted RCT: past tense)\n",
        "#tokenlization and lemma"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RHXD9XjYvU0l",
        "outputId": "818cd9c4-04c2-4213-9536-f8f1e753cc9a"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "PuvCawy2VbRy"
      },
      "source": [
        "import pandas as pd\n",
        "import re\n",
        "import pickle\n",
        "import spacy\n",
        "import numpy as np\n",
        "from spacy.lang.en import English"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jpoYT6Z4VbR9"
      },
      "source": [
        "spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sSYRNkkbVbR-"
      },
      "source": [
        "nlp = spacy.load('en')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YZDVpIa2VbR-"
      },
      "source": [
        "nlp.Defaults.stop_words -= {\"will\",\"had\",\"was\",\"were\",\"did\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cHSpWFYUVbSA"
      },
      "source": [
        "new_stopwords = spacy_stopwords"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "-jAJhwCiVbSC"
      },
      "source": [
        "def preprocess_text(df):\n",
        "    df.AbstractPlusTitle = df.AbstractPlusTitle.fillna(' ')\n",
        "    df.AbstractPlusTitle = df.AbstractPlusTitle.str.replace('\\n',' ')\n",
        "    df.AbstractPlusTitle = df.AbstractPlusTitle.str.replace('\\r',' ')\n",
        "    return df"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "A7mV3c8dVbSH"
      },
      "source": [
        "df = pd.read_pickle('/content/drive/MyDrive/HS651_CAMs_systematic_review_automation/AugmDataset_addTil0.pkl')\n",
        "#df_comb = df_comb.reset_index(drop =  True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "8uIeE7v-FNS5"
      },
      "source": [
        "df = df.reset_index()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 305
        },
        "id": "q1Bqo53gwUEP",
        "outputId": "3d236dca-7562-499a-ee2a-9b82e94ac82e"
      },
      "source": [
        "df.head()"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>index</th>\n",
              "      <th>Title</th>\n",
              "      <th>JournalInfo</th>\n",
              "      <th>Abstract</th>\n",
              "      <th>TiL</th>\n",
              "      <th>AbL</th>\n",
              "      <th>AbstractPlusTitle</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>Qigong or Tai Chi in Cancer Care: an Updated S...</td>\n",
              "      <td>Curr Oncol Rep</td>\n",
              "      <td>Qigong and Tai Chi are two increasingly popula...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Qigong and Tai Chi are two increasingly popula...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>2</td>\n",
              "      <td>Tai Chi and Qigong for cancer-related symptoms...</td>\n",
              "      <td>J Cancer Surviv</td>\n",
              "      <td>This study aims to summarize and critically ev...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>This study aims to summarize and critically ev...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>3</td>\n",
              "      <td>The effectiveness of tai chi in breast cancer ...</td>\n",
              "      <td>Complement Ther Clin Pract</td>\n",
              "      <td>Tai chi has been suggested as a potential effe...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Tai chi has been suggested as a potential effe...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>6</td>\n",
              "      <td>Effectiveness of Tai Chi on fibromyalgia patie...</td>\n",
              "      <td>Complement Ther Med</td>\n",
              "      <td>To identify empirical evidence on the effectiv...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>To identify empirical evidence on the effectiv...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>7</td>\n",
              "      <td>Does Tai Chi Chuan improve psychological well-...</td>\n",
              "      <td>Medicine Baltimore</td>\n",
              "      <td>Breast cancer is the most prevalent cancer in ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>Breast cancer is the most prevalent cancer in ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "   index  ...                                  AbstractPlusTitle\n",
              "0      0  ...  Qigong and Tai Chi are two increasingly popula...\n",
              "1      2  ...  This study aims to summarize and critically ev...\n",
              "2      3  ...  Tai chi has been suggested as a potential effe...\n",
              "3      6  ...  To identify empirical evidence on the effectiv...\n",
              "4      7  ...  Breast cancer is the most prevalent cancer in ...\n",
              "\n",
              "[5 rows x 7 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3opfBBYUOLGs"
      },
      "source": [
        "df = df[['AbstractPlusTitle','AbL']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XvtiJNFgVbSI",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "outputId": "0feba3dd-d00c-4303-db4c-1ef767495d33"
      },
      "source": [
        "preprocess_text(df)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AbstractPlusTitle</th>\n",
              "      <th>AbL</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Qigong and Tai Chi are two increasingly popula...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>This study aims to summarize and critically ev...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Tai chi has been suggested as a potential effe...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>To identify empirical evidence on the effectiv...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Breast cancer is the most prevalent cancer in ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>542</th>\n",
              "      <td>Fibromyalgia is a condition characterised by c...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>543</th>\n",
              "      <td>Mild cognitive impairment (MCI) is cognitive d...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>544</th>\n",
              "      <td>There is no sufficient evidence on the effecti...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>545</th>\n",
              "      <td>Many of the currently available treatments for...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>546</th>\n",
              "      <td>Chronic fatigue (CF) is unexplained fatigue la...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>547 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                     AbstractPlusTitle  AbL\n",
              "0    Qigong and Tai Chi are two increasingly popula...    0\n",
              "1    This study aims to summarize and critically ev...    0\n",
              "2    Tai chi has been suggested as a potential effe...    0\n",
              "3    To identify empirical evidence on the effectiv...    0\n",
              "4    Breast cancer is the most prevalent cancer in ...    0\n",
              "..                                                 ...  ...\n",
              "542  Fibromyalgia is a condition characterised by c...    1\n",
              "543  Mild cognitive impairment (MCI) is cognitive d...    0\n",
              "544  There is no sufficient evidence on the effecti...    0\n",
              "545  Many of the currently available treatments for...    0\n",
              "546  Chronic fatigue (CF) is unexplained fatigue la...    1\n",
              "\n",
              "[547 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hXihc6auVbSJ"
      },
      "source": [
        "def clean_text(x):\n",
        "  #x=re.sub(r\"\\'ll\",\" will \",x) ----  In academic paper, people tend not to use contraction\n",
        "  #x=re.sub('[^a-zA-Z0-9]', ' ',x).lower()\n",
        "  x = re.sub('[^a-zA-Z]', ' ',x).lower()\n",
        "  x = x.split()\n",
        "  new_list = []\n",
        "  for word in x:\n",
        "    if word not in new_stopwords: \n",
        "      new_list.append(word)\n",
        "  return \" \".join(new_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_WovmbsIVbSJ"
      },
      "source": [
        "df['AbstractPlusTitle'] = df.AbstractPlusTitle.apply(lambda x:clean_text(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vk3ctSuQVbSK"
      },
      "source": [
        "#df_comb[df_comb.index.duplicated(keep = False)]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k0uGYncZVbSK"
      },
      "source": [
        "X = df.loc[:,'AbstractPlusTitle']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mclxZhaxVbSK",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6da744d3-1f21-4a70-c66f-e2d762da947f"
      },
      "source": [
        "X"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0      qigong tai chi increasingly popular mind body ...\n",
              "1      study aims summarize critically evaluate effec...\n",
              "2      tai chi suggested potential effective interven...\n",
              "3      identify empirical evidence effectiveness tai ...\n",
              "4      breast cancer prevalent cancer women worldwide...\n",
              "                             ...                        \n",
              "542    fibromyalgia condition characterised chronic w...\n",
              "543    mild cognitive impairment mci cognitive declin...\n",
              "544    sufficient evidence effectiveness acupuncture ...\n",
              "545    currently available treatments obsessive compu...\n",
              "546    chronic fatigue cf unexplained fatigue lasting...\n",
              "Name: AbstractPlusTitle, Length: 547, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DDf3j1nmVbSL"
      },
      "source": [
        "y = df.loc[:,'AbL']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gpHBPXPlVbSL",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "c009747b-6c9e-424d-96da-5f2d3149e217"
      },
      "source": [
        "y"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0      0\n",
              "1      0\n",
              "2      0\n",
              "3      0\n",
              "4      0\n",
              "      ..\n",
              "542    1\n",
              "543    0\n",
              "544    0\n",
              "545    0\n",
              "546    1\n",
              "Name: AbL, Length: 547, dtype: int64"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4gtzjJmeWYV1"
      },
      "source": [
        "df.to_pickle('/content/drive/MyDrive/preprodf_Augm.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6CC3ywdK1dtQ"
      },
      "source": [
        "#df['Abstract'] = df.Abstract.apply(lambda x: x.split())"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WQQyssglykPf"
      },
      "source": [
        "#df['Length'] = df.Abstract.apply(lambda x: len(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "v1lC-kk41OBM"
      },
      "source": [
        "#df.describe()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "c2ou0AIsVbSa"
      },
      "source": [
        "# DistillBERTs"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "pkVE-wvHVbSa",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "14ec5a58-8edd-436f-f4a1-20105fffdbbe"
      },
      "source": [
        "!pip3 install --upgrade tensorflow"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already up-to-date: tensorflow in /usr/local/lib/python3.7/dist-packages (2.4.1)\n",
            "Requirement already satisfied, skipping upgrade: grpcio~=1.32.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.32.0)\n",
            "Requirement already satisfied, skipping upgrade: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.12.0)\n",
            "Requirement already satisfied, skipping upgrade: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.3.0)\n",
            "Requirement already satisfied, skipping upgrade: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.2)\n",
            "Requirement already satisfied, skipping upgrade: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.15.0)\n",
            "Requirement already satisfied, skipping upgrade: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.12)\n",
            "Requirement already satisfied, skipping upgrade: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied, skipping upgrade: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.36.2)\n",
            "Requirement already satisfied, skipping upgrade: h5py~=2.10.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.10.0)\n",
            "Requirement already satisfied, skipping upgrade: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.6.3)\n",
            "Requirement already satisfied, skipping upgrade: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.7.4.3)\n",
            "Requirement already satisfied, skipping upgrade: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.19.5)\n",
            "Requirement already satisfied, skipping upgrade: tensorboard~=2.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.4.1)\n",
            "Requirement already satisfied, skipping upgrade: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.12.1)\n",
            "Requirement already satisfied, skipping upgrade: tensorflow-estimator<2.5.0,>=2.4.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (2.4.0)\n",
            "Requirement already satisfied, skipping upgrade: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.12.4)\n",
            "Requirement already satisfied, skipping upgrade: gast==0.3.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.3.3)\n",
            "Requirement already satisfied, skipping upgrade: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow) (56.0.0)\n",
            "Requirement already satisfied, skipping upgrade: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow) (1.8.0)\n",
            "Requirement already satisfied, skipping upgrade: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow) (0.4.4)\n",
            "Requirement already satisfied, skipping upgrade: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow) (1.28.1)\n",
            "Requirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow) (1.0.1)\n",
            "Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.4->tensorflow) (3.3.4)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow) (2.10)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow) (2020.12.5)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.4->tensorflow) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow) (1.3.0)\n",
            "Requirement already satisfied, skipping upgrade: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow) (4.7.2)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow) (0.2.8)\n",
            "Requirement already satisfied, skipping upgrade: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow) (4.2.1)\n",
            "Requirement already satisfied, skipping upgrade: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.4->tensorflow) (3.10.1)\n",
            "Requirement already satisfied, skipping upgrade: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.4->tensorflow) (3.1.0)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1>=0.1.3 in /usr/local/lib/python3.7/dist-packages (from rsa<5,>=3.1.4; python_version >= \"3.6\"->google-auth<2,>=1.6.3->tensorboard~=2.4->tensorflow) (0.4.8)\n",
            "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.4->tensorflow) (3.4.1)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "nZ39jYB1VbSb",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "e0e51b76-c404-4c7d-816e-8a899410ffe9"
      },
      "source": [
        "!pip install ktrain"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting ktrain\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/67/31cab9d7c0e23333aebc28b082659c1528f9ab7e22d00e7237efe4fc14f6/ktrain-0.26.2.tar.gz (25.3MB)\n",
            "\u001b[K     |████████████████████████████████| 25.3MB 124kB/s \n",
            "\u001b[?25hCollecting scikit-learn==0.23.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f4/cb/64623369f348e9bfb29ff898a57ac7c91ed4921f228e9726546614d63ccb/scikit_learn-0.23.2-cp37-cp37m-manylinux1_x86_64.whl (6.8MB)\n",
            "\u001b[K     |████████████████████████████████| 6.8MB 50.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from ktrain) (3.2.2)\n",
            "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.1.5)\n",
            "Requirement already satisfied: fastprogress>=0.1.21 in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.0.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from ktrain) (2.23.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.0.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from ktrain) (20.9)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.7/dist-packages (from ktrain) (5.5.0)\n",
            "Collecting langdetect\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/56/a3/8407c1e62d5980188b4acc45ef3d94b933d14a2ebc9ef3505f22cf772570/langdetect-1.0.8.tar.gz (981kB)\n",
            "\u001b[K     |████████████████████████████████| 983kB 46.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: jieba in /usr/local/lib/python3.7/dist-packages (from ktrain) (0.42.1)\n",
            "Collecting cchardet\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/80/72/a4fba7559978de00cf44081c548c5d294bf00ac7dcda2db405d2baa8c67a/cchardet-2.1.7-cp37-cp37m-manylinux2010_x86_64.whl (263kB)\n",
            "\u001b[K     |████████████████████████████████| 266kB 52.5MB/s \n",
            "\u001b[?25hCollecting syntok\n",
            "  Downloading https://files.pythonhosted.org/packages/8c/76/a49e73a04b3e3a14ce232e8e28a1587f8108baa665644fe8c40e307e792e/syntok-1.3.1.tar.gz\n",
            "Collecting seqeval==0.0.19\n",
            "  Downloading https://files.pythonhosted.org/packages/93/e5/b7705156a77f742cfe4fc6f22d0c71591edb2d243328dff2f8fc0f933ab6/seqeval-0.0.19.tar.gz\n",
            "Collecting transformers<=4.3.3,>=4.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/54/5ca07ec9569d2f232f3166de5457b63943882f7950ddfcc887732fc7fb23/transformers-4.3.3-py3-none-any.whl (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 47.9MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 48.7MB/s \n",
            "\u001b[?25hCollecting keras_bert>=0.86.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e2/7f/95fabd29f4502924fa3f09ff6538c5a7d290dfef2c2fe076d3d1a16e08f0/keras-bert-0.86.0.tar.gz\n",
            "Requirement already satisfied: networkx>=2.3 in /usr/local/lib/python3.7/dist-packages (from ktrain) (2.5.1)\n",
            "Collecting whoosh\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ba/19/24d0f1f454a2c1eb689ca28d2f178db81e5024f42d82729a4ff6771155cf/Whoosh-2.7.4-py2.py3-none-any.whl (468kB)\n",
            "\u001b[K     |████████████████████████████████| 471kB 35.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.2->ktrain) (1.4.1)\n",
            "Requirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.2->ktrain) (1.19.5)\n",
            "Collecting threadpoolctl>=2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f7/12/ec3f2e203afa394a149911729357aa48affc59c20e2c1c8297a60f33f133/threadpoolctl-2.1.0-py3-none-any.whl\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (2.8.1)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (2.4.7)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (0.10.0)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (1.3.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.1->ktrain) (2018.9)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (1.24.3)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (0.8.1)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (5.0.5)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (4.8.0)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (1.0.18)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (56.0.0)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (4.4.2)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (2.6.1)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (0.7.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from langdetect->ktrain) (1.15.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from syntok->ktrain) (2019.12.20)\n",
            "Requirement already satisfied: Keras>=2.2.4 in /usr/local/lib/python3.7/dist-packages (from seqeval==0.0.19->ktrain) (2.4.3)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (4.41.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (3.0.12)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 48.7MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 59.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (3.10.1)\n",
            "Collecting keras-transformer>=0.38.0\n",
            "  Downloading https://files.pythonhosted.org/packages/89/6c/d6f0c164f4cc16fbc0d0fea85f5526e87a7d2df7b077809e422a7e626150/keras-transformer-0.38.0.tar.gz\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from traitlets>=4.2->ipython->ktrain) (0.2.0)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect; sys_platform != \"win32\"->ipython->ktrain) (0.7.0)\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->ktrain) (0.2.5)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from Keras>=2.2.4->seqeval==0.0.19->ktrain) (3.13)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from Keras>=2.2.4->seqeval==0.0.19->ktrain) (2.10.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<=4.3.3,>=4.0.0->ktrain) (7.1.2)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<=4.3.3,>=4.0.0->ktrain) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<=4.3.3,>=4.0.0->ktrain) (3.4.1)\n",
            "Collecting keras-pos-embd>=0.11.0\n",
            "  Downloading https://files.pythonhosted.org/packages/09/70/b63ed8fc660da2bb6ae29b9895401c628da5740c048c190b5d7107cadd02/keras-pos-embd-0.11.0.tar.gz\n",
            "Collecting keras-multi-head>=0.27.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e6/32/45adf2549450aca7867deccfa04af80a0ab1ca139af44b16bc669e0e09cd/keras-multi-head-0.27.0.tar.gz\n",
            "Collecting keras-layer-normalization>=0.14.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a4/0e/d1078df0494bac9ce1a67954e5380b6e7569668f0f3b50a9531c62c1fc4a/keras-layer-normalization-0.14.0.tar.gz\n",
            "Collecting keras-position-wise-feed-forward>=0.6.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e3/59/f0faa1037c033059e7e9e7758e6c23b4d1c0772cd48de14c4b6fd4033ad5/keras-position-wise-feed-forward-0.6.0.tar.gz\n",
            "Collecting keras-embed-sim>=0.8.0\n",
            "  Downloading https://files.pythonhosted.org/packages/57/ef/61a1e39082c9e1834a2d09261d4a0b69f7c818b359216d4e1912b20b1c86/keras-embed-sim-0.8.0.tar.gz\n",
            "Collecting keras-self-attention==0.46.0\n",
            "  Downloading https://files.pythonhosted.org/packages/15/6b/c804924a056955fa1f3ff767945187103cfc851ba9bd0fc5a6c6bc18e2eb/keras-self-attention-0.46.0.tar.gz\n",
            "Building wheels for collected packages: ktrain, langdetect, syntok, seqeval, keras-bert, keras-transformer, keras-pos-embd, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-self-attention\n",
            "  Building wheel for ktrain (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ktrain: filename=ktrain-0.26.2-cp37-none-any.whl size=25277794 sha256=20736a8c370aaeeef91368a3e942d1da6eb65a4d827476e3f4fdd715e20e1567\n",
            "  Stored in directory: /root/.cache/pip/wheels/67/2e/f1/c72afa08df8b2d984b910dea228902ce81dae4511afe9fafd2\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.8-cp37-none-any.whl size=993193 sha256=2244acb7ecfb70b72cfb60c48e7ea8625e2c56a9972819ee2fd2fb563ddbb243\n",
            "  Stored in directory: /root/.cache/pip/wheels/8d/b3/aa/6d99de9f3841d7d3d40a60ea06e6d669e8e5012e6c8b947a57\n",
            "  Building wheel for syntok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for syntok: filename=syntok-1.3.1-cp37-none-any.whl size=20919 sha256=14ea8209f4670f151899b856a33779641faa654ba3df814e0a30bc23fd62daa9\n",
            "  Stored in directory: /root/.cache/pip/wheels/51/c6/a4/be1920586c49469846bcd2888200bdecfe109ec421dab9be2d\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-0.0.19-cp37-none-any.whl size=9919 sha256=b97ba41ec7a7896ce3f7f9aaa6bb65999a81b84b56e24231ce81bfffc9a92f06\n",
            "  Stored in directory: /root/.cache/pip/wheels/8d/1f/bf/1198beceed805a2099060975f6281d1b01046dd279e19c97be\n",
            "  Building wheel for keras-bert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-bert: filename=keras_bert-0.86.0-cp37-none-any.whl size=34144 sha256=9a0d28053766691a9300978e11040346037dbbd6456879b6f2a67a93ec6c32be\n",
            "  Stored in directory: /root/.cache/pip/wheels/66/f0/b1/748128b58562fc9e31b907bb5e2ab6a35eb37695e83911236b\n",
            "  Building wheel for keras-transformer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-transformer: filename=keras_transformer-0.38.0-cp37-none-any.whl size=12942 sha256=fc706e74ed74d528c3fa4976d7e8b8bb8319a95a511f7b50eb9c2654d89a4673\n",
            "  Stored in directory: /root/.cache/pip/wheels/e5/fb/3a/37b2b9326c799aa010ae46a04ddb04f320d8c77c0b7e837f4e\n",
            "  Building wheel for keras-pos-embd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-pos-embd: filename=keras_pos_embd-0.11.0-cp37-none-any.whl size=7554 sha256=c926745d31ce93e02ddd2c00ed61d667ff4225bf6062baaeae6a66e6ed5c1871\n",
            "  Stored in directory: /root/.cache/pip/wheels/5b/a1/a0/ce6b1d49ba1a9a76f592e70cf297b05c96bc9f418146761032\n",
            "  Building wheel for keras-multi-head (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-multi-head: filename=keras_multi_head-0.27.0-cp37-none-any.whl size=15611 sha256=482be79742fe49cd2a02f966adc15a160788fb13eaeb708321e2b86cfe620761\n",
            "  Stored in directory: /root/.cache/pip/wheels/b5/b4/49/0a0c27dcb93c13af02fea254ff51d1a43a924dd4e5b7a7164d\n",
            "  Building wheel for keras-layer-normalization (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-layer-normalization: filename=keras_layer_normalization-0.14.0-cp37-none-any.whl size=5269 sha256=b36a5a8f40f68ea43ab95e91fe1728a65e0df66dd5de7bdda4b6eb82c57aaf3c\n",
            "  Stored in directory: /root/.cache/pip/wheels/54/80/22/a638a7d406fd155e507aa33d703e3fa2612b9eb7bb4f4fe667\n",
            "  Building wheel for keras-position-wise-feed-forward (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-position-wise-feed-forward: filename=keras_position_wise_feed_forward-0.6.0-cp37-none-any.whl size=5623 sha256=9c04f280f34c18d58e5439ea7b569f05f466aef2e87a152e6ce01e43555b1f0b\n",
            "  Stored in directory: /root/.cache/pip/wheels/39/e2/e2/3514fef126a00574b13bc0b9e23891800158df3a3c19c96e3b\n",
            "  Building wheel for keras-embed-sim (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-embed-sim: filename=keras_embed_sim-0.8.0-cp37-none-any.whl size=4558 sha256=ba8cbe1c8c67d381049a86e1a183e31d7b93ad6bb985537daad099a33c499cb3\n",
            "  Stored in directory: /root/.cache/pip/wheels/49/45/8b/c111f6cc8bec253e984677de73a6f4f5d2f1649f42aac191c8\n",
            "  Building wheel for keras-self-attention (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-self-attention: filename=keras_self_attention-0.46.0-cp37-none-any.whl size=17278 sha256=d7d5ef167b0007a85e802d09093a8576c4ac3e21a0702b26b4474a9c948c7ae9\n",
            "  Stored in directory: /root/.cache/pip/wheels/d2/2e/80/fec4c05eb23c8e13b790e26d207d6e0ffe8013fad8c6bdd4d2\n",
            "Successfully built ktrain langdetect syntok seqeval keras-bert keras-transformer keras-pos-embd keras-multi-head keras-layer-normalization keras-position-wise-feed-forward keras-embed-sim keras-self-attention\n",
            "Installing collected packages: threadpoolctl, scikit-learn, langdetect, cchardet, syntok, seqeval, tokenizers, sacremoses, transformers, sentencepiece, keras-pos-embd, keras-self-attention, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-transformer, keras-bert, whoosh, ktrain\n",
            "  Found existing installation: scikit-learn 0.22.2.post1\n",
            "    Uninstalling scikit-learn-0.22.2.post1:\n",
            "      Successfully uninstalled scikit-learn-0.22.2.post1\n",
            "Successfully installed cchardet-2.1.7 keras-bert-0.86.0 keras-embed-sim-0.8.0 keras-layer-normalization-0.14.0 keras-multi-head-0.27.0 keras-pos-embd-0.11.0 keras-position-wise-feed-forward-0.6.0 keras-self-attention-0.46.0 keras-transformer-0.38.0 ktrain-0.26.2 langdetect-1.0.8 sacremoses-0.0.45 scikit-learn-0.23.2 sentencepiece-0.1.95 seqeval-0.0.19 syntok-1.3.1 threadpoolctl-2.1.0 tokenizers-0.10.2 transformers-4.3.3 whoosh-2.7.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "MkZEhwFCCqQf"
      },
      "source": [
        "df = pd.read_pickle('/content/drive/MyDrive/preprodf_Augm.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "SZqWVHlBVbSb"
      },
      "source": [
        "df_valid = df.sample(frac = 0.2, random_state = 42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "X7RboAbLIGog",
        "outputId": "bf9af342-ea7a-4f3d-f1b2-8674efd8d71c"
      },
      "source": [
        "type(df_valid)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pandas.core.frame.DataFrame"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 61
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pmzu45oEVbSc"
      },
      "source": [
        "#df_valid.to_pickle('/content/drive/MyDrive/AbsTil_valid.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bsdnFvHPVbSc"
      },
      "source": [
        "df_train = df.drop(df_valid.index)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2oWVgOSMOsnC",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 417
        },
        "outputId": "0c7a8b9d-27e4-441c-acaf-d0a38beb18b4"
      },
      "source": [
        "df_train"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AbstractPlusTitle</th>\n",
              "      <th>AbL</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>study aims summarize critically evaluate effec...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>identify empirical evidence effectiveness tai ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>breast cancer prevalent cancer women worldwide...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5</th>\n",
              "      <td>post stroke mental disorders psmds post stroke...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7</th>\n",
              "      <td>insomnia prevalent significant public health c...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>541</th>\n",
              "      <td>study participants female fibromyalgia fm were...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>543</th>\n",
              "      <td>mild cognitive impairment mci cognitive declin...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>544</th>\n",
              "      <td>sufficient evidence effectiveness acupuncture ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>545</th>\n",
              "      <td>currently available treatments obsessive compu...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>546</th>\n",
              "      <td>chronic fatigue cf unexplained fatigue lasting...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>438 rows × 2 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                     AbstractPlusTitle  AbL\n",
              "1    study aims summarize critically evaluate effec...    0\n",
              "3    identify empirical evidence effectiveness tai ...    0\n",
              "4    breast cancer prevalent cancer women worldwide...    0\n",
              "5    post stroke mental disorders psmds post stroke...    0\n",
              "7    insomnia prevalent significant public health c...    0\n",
              "..                                                 ...  ...\n",
              "541  study participants female fibromyalgia fm were...    0\n",
              "543  mild cognitive impairment mci cognitive declin...    0\n",
              "544  sufficient evidence effectiveness acupuncture ...    0\n",
              "545  currently available treatments obsessive compu...    0\n",
              "546  chronic fatigue cf unexplained fatigue lasting...    1\n",
              "\n",
              "[438 rows x 2 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o47_Ite8KXYK",
        "outputId": "a8db6be8-a9a7-4cce-a790-0056c45f0399"
      },
      "source": [
        "type(df_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "pandas.core.frame.DataFrame"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 73
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9EFC-4XqVbSc"
      },
      "source": [
        "#df_train.to_pickle('/content/drive/MyDrive/AbsTil_train.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BpX2oKjko-Hj"
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "import matplotlib.pyplot as plt\n",
        "import pickle"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sSVqixArVbSc"
      },
      "source": [
        "#df_train = pd.read_pickle('/content/drive/MyDrive/HS651_CAMs_systematic_review_automation/BOW_Distill_BertS/AbsTil_train.pkl')\n",
        "#df_valid = pd.read_pickle('/content/drive/MyDrive/HS651_CAMs_systematic_review_automation/BOW_Distill_BertS/AbsTil_valid.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "dzg4rZjCo7Lp"
      },
      "source": [
        "import tensorflow as tf\n",
        "from sklearn import metrics\n",
        "import ktrain\n",
        "from ktrain import text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "h6M6CqfcKgTz",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "926d9e0e-4a75-4bbd-9177-614d788f2da3"
      },
      "source": [
        "train_sizes = [10,50,100,200,300]\n",
        "for size in train_sizes:\n",
        "    #print(type(size))\n",
        "     df_train = df.drop(df_valid.index)\n",
        "     df_train = df_train.sample(n=size)\n",
        "     print(type(df_train))"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "<class 'pandas.core.frame.DataFrame'>\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "<class 'pandas.core.frame.DataFrame'>\n",
            "<class 'pandas.core.frame.DataFrame'>\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "IGVUoQxermNU",
        "outputId": "3153f47d-d2f5-4f68-ddc8-0e04ea0d7aec"
      },
      "source": [
        "import ktrain\n",
        "from ktrain import text\n",
        "# df = pd.read_pickle('/content/drive/MyDrive/preprodf_Augm.pkl')\n",
        "# df_valid = df.sample(frac = 0.2, random_state = 42)\n",
        "train_sizes = [10,50,100,200,300]\n",
        "df_train = df.drop(df_valid.index)\n",
        "for size in train_sizes:\n",
        "#   df_train = df.drop(df_valid.index)\n",
        "    df_tra = df_train.sample(n=size)\n",
        "    (train, val, preproc) = text.texts_from_df(train_df=df_tra, \n",
        "                                               text_column='AbstractPlusTitle', \n",
        "                                               label_columns='AbL',\n",
        "                                               val_df = df_valid,\n",
        "                                               maxlen = 512, #length of each doc aligned to 512\n",
        "                                               preprocess_mode = 'distilbert')\n",
        "    bertmodel = text.text_classifier(name = 'distilbert', train_data = train, preproc=preproc)\n",
        "    learner = ktrain.get_learner(model = bertmodel,\n",
        "                             train_data = train,\n",
        "                             val_data = val,\n",
        "                             batch_size = 5)\n",
        "    learner.fit_onecycle(lr = 2e-5, epochs=10)\n",
        "    predictor = ktrain.get_predictor(learner.model, preproc)\n",
        "    textval=df_valid['AbstractPlusTitle']\n",
        "    data=list(textval)\n",
        "    y_pred = predictor.predict(data, return_proba=True)\n",
        "    df_valid.AbL = df_valid.AbL.astype('int')\n",
        "    df_tra.AbL = df_tra.AbL.astype('int')\n",
        "    y_test = df_valid['AbL']\n",
        "    fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred[:,1])\n",
        "    auc = metrics.auc(fpr, tpr)\n",
        "    print(auc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "230      0.0  1.0\n",
            "125      1.0  0.0\n",
            "541      1.0  0.0\n",
            "284      0.0  1.0\n",
            "296      0.0  1.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 162\n",
            "\t95percentile : 203\n",
            "\t99percentile : 205\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "2/2 [==============================] - 11s 4s/step - loss: 0.6766 - accuracy: 0.6000 - val_loss: 0.6829 - val_accuracy: 0.6055\n",
            "Epoch 2/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6988 - accuracy: 0.3333 - val_loss: 0.6820 - val_accuracy: 0.6055\n",
            "Epoch 3/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6898 - accuracy: 0.4667 - val_loss: 0.6820 - val_accuracy: 0.6055\n",
            "Epoch 4/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.7095 - accuracy: 0.4000 - val_loss: 0.6816 - val_accuracy: 0.6055\n",
            "Epoch 5/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6706 - accuracy: 0.4667 - val_loss: 0.6826 - val_accuracy: 0.6055\n",
            "Epoch 6/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6699 - accuracy: 0.8000 - val_loss: 0.6836 - val_accuracy: 0.6422\n",
            "Epoch 7/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6710 - accuracy: 0.6667 - val_loss: 0.6820 - val_accuracy: 0.6330\n",
            "Epoch 8/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6439 - accuracy: 0.9333 - val_loss: 0.6804 - val_accuracy: 0.6239\n",
            "Epoch 9/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6423 - accuracy: 0.9333 - val_loss: 0.6787 - val_accuracy: 0.6239\n",
            "Epoch 10/10\n",
            "2/2 [==============================] - 3s 3s/step - loss: 0.6399 - accuracy: 1.0000 - val_loss: 0.6780 - val_accuracy: 0.6239\n",
            "0.700845665961945\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "279      1.0  0.0\n",
            "516      0.0  1.0\n",
            "284      0.0  1.0\n",
            "1        1.0  0.0\n",
            "394      1.0  0.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 205\n",
            "\t95percentile : 358\n",
            "\t99percentile : 499\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "10/10 [==============================] - 14s 718ms/step - loss: 0.6844 - accuracy: 0.6791 - val_loss: 0.6803 - val_accuracy: 0.6055\n",
            "Epoch 2/10\n",
            "10/10 [==============================] - 5s 563ms/step - loss: 0.6890 - accuracy: 0.5420 - val_loss: 0.6771 - val_accuracy: 0.6055\n",
            "Epoch 3/10\n",
            "10/10 [==============================] - 5s 571ms/step - loss: 0.6752 - accuracy: 0.6122 - val_loss: 0.6687 - val_accuracy: 0.6055\n",
            "Epoch 4/10\n",
            "10/10 [==============================] - 6s 582ms/step - loss: 0.6846 - accuracy: 0.5079 - val_loss: 0.6618 - val_accuracy: 0.6055\n",
            "Epoch 5/10\n",
            "10/10 [==============================] - 6s 587ms/step - loss: 0.6551 - accuracy: 0.5344 - val_loss: 0.6508 - val_accuracy: 0.6055\n",
            "Epoch 6/10\n",
            "10/10 [==============================] - 6s 587ms/step - loss: 0.6126 - accuracy: 0.7151 - val_loss: 0.5991 - val_accuracy: 0.6422\n",
            "Epoch 7/10\n",
            "10/10 [==============================] - 6s 580ms/step - loss: 0.4645 - accuracy: 0.8358 - val_loss: 0.5653 - val_accuracy: 0.6881\n",
            "Epoch 8/10\n",
            "10/10 [==============================] - 6s 577ms/step - loss: 0.3539 - accuracy: 0.9093 - val_loss: 0.5522 - val_accuracy: 0.6972\n",
            "Epoch 9/10\n",
            "10/10 [==============================] - 5s 574ms/step - loss: 0.3371 - accuracy: 0.9076 - val_loss: 0.5685 - val_accuracy: 0.7064\n",
            "Epoch 10/10\n",
            "10/10 [==============================] - 5s 570ms/step - loss: 0.2624 - accuracy: 0.9895 - val_loss: 0.5757 - val_accuracy: 0.7156\n",
            "0.828752642706131\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "300      0.0  1.0\n",
            "436      0.0  1.0\n",
            "201      0.0  1.0\n",
            "539      0.0  1.0\n",
            "138      1.0  0.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 180\n",
            "\t95percentile : 263\n",
            "\t99percentile : 421\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "20/20 [==============================] - 18s 501ms/step - loss: 0.6951 - accuracy: 0.4917 - val_loss: 0.6928 - val_accuracy: 0.5321\n",
            "Epoch 2/10\n",
            "20/20 [==============================] - 9s 436ms/step - loss: 0.6906 - accuracy: 0.4989 - val_loss: 0.6831 - val_accuracy: 0.7615\n",
            "Epoch 3/10\n",
            "20/20 [==============================] - 9s 445ms/step - loss: 0.6803 - accuracy: 0.5890 - val_loss: 0.6756 - val_accuracy: 0.5596\n",
            "Epoch 4/10\n",
            "20/20 [==============================] - 9s 456ms/step - loss: 0.6697 - accuracy: 0.6491 - val_loss: 0.5780 - val_accuracy: 0.8073\n",
            "Epoch 5/10\n",
            "20/20 [==============================] - 9s 450ms/step - loss: 0.5092 - accuracy: 0.8635 - val_loss: 0.4810 - val_accuracy: 0.8165\n",
            "Epoch 6/10\n",
            "20/20 [==============================] - 9s 440ms/step - loss: 0.2862 - accuracy: 0.9436 - val_loss: 0.4266 - val_accuracy: 0.8349\n",
            "Epoch 7/10\n",
            "20/20 [==============================] - 9s 435ms/step - loss: 0.1779 - accuracy: 0.9717 - val_loss: 0.3372 - val_accuracy: 0.8532\n",
            "Epoch 8/10\n",
            "20/20 [==============================] - 9s 433ms/step - loss: 0.0849 - accuracy: 0.9870 - val_loss: 0.3628 - val_accuracy: 0.8532\n",
            "Epoch 9/10\n",
            "20/20 [==============================] - 9s 431ms/step - loss: 0.0638 - accuracy: 0.9911 - val_loss: 0.3481 - val_accuracy: 0.8716\n",
            "Epoch 10/10\n",
            "20/20 [==============================] - 9s 432ms/step - loss: 0.0428 - accuracy: 1.0000 - val_loss: 0.3735 - val_accuracy: 0.8716\n",
            "0.9422128259337561\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "114      1.0  0.0\n",
            "110      1.0  0.0\n",
            "421      0.0  1.0\n",
            "221      0.0  1.0\n",
            "352      0.0  1.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 201\n",
            "\t95percentile : 387\n",
            "\t99percentile : 588\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "40/40 [==============================] - 24s 409ms/step - loss: 0.6853 - accuracy: 0.5393 - val_loss: 0.6669 - val_accuracy: 0.6055\n",
            "Epoch 2/10\n",
            "40/40 [==============================] - 15s 388ms/step - loss: 0.6750 - accuracy: 0.5612 - val_loss: 0.6553 - val_accuracy: 0.6055\n",
            "Epoch 3/10\n",
            "40/40 [==============================] - 15s 382ms/step - loss: 0.6604 - accuracy: 0.5805 - val_loss: 0.6124 - val_accuracy: 0.8073\n",
            "Epoch 4/10\n",
            "40/40 [==============================] - 15s 371ms/step - loss: 0.6313 - accuracy: 0.6671 - val_loss: 0.6778 - val_accuracy: 0.5505\n",
            "Epoch 5/10\n",
            "40/40 [==============================] - 15s 369ms/step - loss: 0.5731 - accuracy: 0.6799 - val_loss: 0.3836 - val_accuracy: 0.8349\n",
            "Epoch 6/10\n",
            "40/40 [==============================] - 15s 373ms/step - loss: 0.4819 - accuracy: 0.7608 - val_loss: 0.3782 - val_accuracy: 0.8257\n",
            "Epoch 7/10\n",
            "40/40 [==============================] - 15s 378ms/step - loss: 0.3058 - accuracy: 0.8945 - val_loss: 0.3899 - val_accuracy: 0.8624\n",
            "Epoch 8/10\n",
            "40/40 [==============================] - 15s 378ms/step - loss: 0.2828 - accuracy: 0.8770 - val_loss: 0.3858 - val_accuracy: 0.8440\n",
            "Epoch 9/10\n",
            "40/40 [==============================] - 15s 376ms/step - loss: 0.2057 - accuracy: 0.9208 - val_loss: 0.4367 - val_accuracy: 0.8440\n",
            "Epoch 10/10\n",
            "40/40 [==============================] - 15s 374ms/step - loss: 0.1256 - accuracy: 0.9752 - val_loss: 0.4151 - val_accuracy: 0.8257\n",
            "0.9154334038054968\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "450      0.0  1.0\n",
            "241      0.0  1.0\n",
            "240      1.0  0.0\n",
            "321      1.0  0.0\n",
            "52       1.0  0.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 191\n",
            "\t95percentile : 319\n",
            "\t99percentile : 557\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "60/60 [==============================] - 30s 377ms/step - loss: 0.6880 - accuracy: 0.5549 - val_loss: 0.6587 - val_accuracy: 0.6055\n",
            "Epoch 2/10\n",
            "60/60 [==============================] - 22s 364ms/step - loss: 0.6618 - accuracy: 0.5878 - val_loss: 0.5667 - val_accuracy: 0.6514\n",
            "Epoch 3/10\n",
            "60/60 [==============================] - 21s 353ms/step - loss: 0.5379 - accuracy: 0.7196 - val_loss: 0.4814 - val_accuracy: 0.7798\n",
            "Epoch 4/10\n",
            "60/60 [==============================] - 21s 350ms/step - loss: 0.4013 - accuracy: 0.8531 - val_loss: 0.3540 - val_accuracy: 0.8532\n",
            "Epoch 5/10\n",
            "60/60 [==============================] - 21s 354ms/step - loss: 0.2400 - accuracy: 0.9279 - val_loss: 0.3589 - val_accuracy: 0.8716\n",
            "Epoch 6/10\n",
            "60/60 [==============================] - 21s 355ms/step - loss: 0.2006 - accuracy: 0.9373 - val_loss: 0.3428 - val_accuracy: 0.8624\n",
            "Epoch 7/10\n",
            "60/60 [==============================] - 21s 353ms/step - loss: 0.0701 - accuracy: 0.9894 - val_loss: 0.4287 - val_accuracy: 0.8257\n",
            "Epoch 8/10\n",
            "60/60 [==============================] - 21s 352ms/step - loss: 0.0395 - accuracy: 0.9942 - val_loss: 0.5326 - val_accuracy: 0.8257\n",
            "Epoch 9/10\n",
            "60/60 [==============================] - 21s 353ms/step - loss: 0.0456 - accuracy: 0.9899 - val_loss: 0.4430 - val_accuracy: 0.8349\n",
            "Epoch 10/10\n",
            "60/60 [==============================] - 21s 353ms/step - loss: 0.0234 - accuracy: 0.9964 - val_loss: 0.4506 - val_accuracy: 0.8440\n",
            "0.92353770260747\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iVeHNambcY8J"
      },
      "source": [
        "#print(tf.__version__)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "fVeIzNgHdAG5",
        "outputId": "bc495bec-ed85-4212-88ee-9c950a8be14b"
      },
      "source": [
        "import ktrain\n",
        "from ktrain import text\n",
        "# df = pd.read_pickle('/content/drive/MyDrive/preprodf_Augm.pkl')\n",
        "# df_valid = df.sample(frac = 0.2, random_state = 42)\n",
        "train_sizes = [50,70,80,100]\n",
        "df_train = df.drop(df_valid.index)\n",
        "for size in train_sizes:\n",
        "#   df_train = df.drop(df_valid.index)\n",
        "    df_tra = df_train.sample(n=size)\n",
        "    (train, val, preproc) = text.texts_from_df(train_df=df_tra, \n",
        "                                               text_column='AbstractPlusTitle', \n",
        "                                               label_columns='AbL',\n",
        "                                               val_df = df_valid,\n",
        "                                               maxlen = 512, #length of each doc aligned to 512\n",
        "                                               preprocess_mode = 'distilbert')\n",
        "    bertmodel = text.text_classifier(name = 'distilbert', train_data = train, preproc=preproc)\n",
        "    learner = ktrain.get_learner(model = bertmodel,\n",
        "                             train_data = train,\n",
        "                             val_data = val,\n",
        "                             batch_size = 5)\n",
        "    learner.fit_onecycle(lr = 2e-5, epochs=10)\n",
        "    predictor = ktrain.get_predictor(learner.model, preproc)\n",
        "    textval=df_valid['AbstractPlusTitle']\n",
        "    data=list(textval)\n",
        "    y_pred = predictor.predict(data, return_proba=True)\n",
        "    df_valid.AbL = df_valid.AbL.astype('int')\n",
        "    df_tra.AbL = df_tra.AbL.astype('int')\n",
        "    y_test = df_valid['AbL']\n",
        "    fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred[:,1])\n",
        "    auc = metrics.auc(fpr, tpr)\n",
        "    print(df_tra.AbL.value_counts()) \n",
        "    print(auc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "315      1.0  0.0\n",
            "200      1.0  0.0\n",
            "508      1.0  0.0\n",
            "541      1.0  0.0\n",
            "135      1.0  0.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 181\n",
            "\t95percentile : 268\n",
            "\t99percentile : 355\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "10/10 [==============================] - 14s 703ms/step - loss: 0.6939 - accuracy: 0.6325 - val_loss: 0.6816 - val_accuracy: 0.6055\n",
            "Epoch 2/10\n",
            "10/10 [==============================] - 5s 553ms/step - loss: 0.6863 - accuracy: 0.5566 - val_loss: 0.6874 - val_accuracy: 0.5963\n",
            "Epoch 3/10\n",
            "10/10 [==============================] - 5s 560ms/step - loss: 0.6890 - accuracy: 0.5191 - val_loss: 0.7041 - val_accuracy: 0.3945\n",
            "Epoch 4/10\n",
            "10/10 [==============================] - 5s 568ms/step - loss: 0.6895 - accuracy: 0.5787 - val_loss: 0.6778 - val_accuracy: 0.6239\n",
            "Epoch 5/10\n",
            "10/10 [==============================] - 5s 576ms/step - loss: 0.6702 - accuracy: 0.6669 - val_loss: 0.6705 - val_accuracy: 0.6789\n",
            "Epoch 6/10\n",
            "10/10 [==============================] - 6s 583ms/step - loss: 0.6342 - accuracy: 0.7537 - val_loss: 0.6328 - val_accuracy: 0.6055\n",
            "Epoch 7/10\n",
            "10/10 [==============================] - 6s 589ms/step - loss: 0.5914 - accuracy: 0.6566 - val_loss: 0.6081 - val_accuracy: 0.7156\n",
            "Epoch 8/10\n",
            "10/10 [==============================] - 6s 596ms/step - loss: 0.4395 - accuracy: 0.8727 - val_loss: 0.5710 - val_accuracy: 0.7156\n",
            "Epoch 9/10\n",
            "10/10 [==============================] - 6s 594ms/step - loss: 0.2856 - accuracy: 0.9339 - val_loss: 0.5819 - val_accuracy: 0.7064\n",
            "Epoch 10/10\n",
            "10/10 [==============================] - 6s 583ms/step - loss: 0.2052 - accuracy: 0.9943 - val_loss: 0.5961 - val_accuracy: 0.7156\n",
            "1    25\n",
            "0    25\n",
            "Name: AbL, dtype: int64\n",
            "0.7607470049330515\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "314      0.0  1.0\n",
            "234      0.0  1.0\n",
            "182      1.0  0.0\n",
            "301      0.0  1.0\n",
            "19       1.0  0.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 189\n",
            "\t95percentile : 386\n",
            "\t99percentile : 515\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "14/14 [==============================] - 16s 596ms/step - loss: 0.6988 - accuracy: 0.3728 - val_loss: 0.6859 - val_accuracy: 0.6147\n",
            "Epoch 2/10\n",
            "14/14 [==============================] - 7s 490ms/step - loss: 0.6823 - accuracy: 0.5255 - val_loss: 0.6666 - val_accuracy: 0.6055\n",
            "Epoch 3/10\n",
            "14/14 [==============================] - 7s 497ms/step - loss: 0.6514 - accuracy: 0.5943 - val_loss: 0.6487 - val_accuracy: 0.6055\n",
            "Epoch 4/10\n",
            "14/14 [==============================] - 7s 505ms/step - loss: 0.6633 - accuracy: 0.5622 - val_loss: 0.6200 - val_accuracy: 0.6055\n",
            "Epoch 5/10\n",
            "14/14 [==============================] - 7s 513ms/step - loss: 0.5912 - accuracy: 0.6961 - val_loss: 0.5412 - val_accuracy: 0.7431\n",
            "Epoch 6/10\n",
            "14/14 [==============================] - 7s 514ms/step - loss: 0.4145 - accuracy: 0.8613 - val_loss: 0.4801 - val_accuracy: 0.7798\n",
            "Epoch 7/10\n",
            "14/14 [==============================] - 7s 506ms/step - loss: 0.2347 - accuracy: 0.9208 - val_loss: 0.5698 - val_accuracy: 0.6789\n",
            "Epoch 8/10\n",
            "14/14 [==============================] - 7s 498ms/step - loss: 0.1778 - accuracy: 0.9537 - val_loss: 0.5539 - val_accuracy: 0.7706\n",
            "Epoch 9/10\n",
            "14/14 [==============================] - 7s 492ms/step - loss: 0.0940 - accuracy: 1.0000 - val_loss: 0.5879 - val_accuracy: 0.7156\n",
            "Epoch 10/10\n",
            "14/14 [==============================] - 7s 489ms/step - loss: 0.1472 - accuracy: 0.9557 - val_loss: 0.5524 - val_accuracy: 0.7798\n",
            "0    42\n",
            "1    28\n",
            "Name: AbL, dtype: int64\n",
            "0.8491895701198027\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "141      1.0  0.0\n",
            "419      0.0  1.0\n",
            "108      1.0  0.0\n",
            "438      0.0  1.0\n",
            "394      1.0  0.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 179\n",
            "\t95percentile : 280\n",
            "\t99percentile : 380\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "16/16 [==============================] - 17s 550ms/step - loss: 0.7006 - accuracy: 0.4158 - val_loss: 0.7021 - val_accuracy: 0.3945\n",
            "Epoch 2/10\n",
            "16/16 [==============================] - 7s 464ms/step - loss: 0.6983 - accuracy: 0.4313 - val_loss: 0.6978 - val_accuracy: 0.3945\n",
            "Epoch 3/10\n",
            "16/16 [==============================] - 7s 473ms/step - loss: 0.6808 - accuracy: 0.5277 - val_loss: 0.7086 - val_accuracy: 0.3945\n",
            "Epoch 4/10\n",
            "16/16 [==============================] - 7s 478ms/step - loss: 0.6857 - accuracy: 0.5087 - val_loss: 0.6815 - val_accuracy: 0.4312\n",
            "Epoch 5/10\n",
            "16/16 [==============================] - 8s 484ms/step - loss: 0.5681 - accuracy: 0.8118 - val_loss: 0.4853 - val_accuracy: 0.8257\n",
            "Epoch 6/10\n",
            "16/16 [==============================] - 8s 483ms/step - loss: 0.3873 - accuracy: 0.8720 - val_loss: 0.4408 - val_accuracy: 0.8165\n",
            "Epoch 7/10\n",
            "16/16 [==============================] - 7s 473ms/step - loss: 0.2942 - accuracy: 0.8878 - val_loss: 0.4892 - val_accuracy: 0.7890\n",
            "Epoch 8/10\n",
            "16/16 [==============================] - 7s 467ms/step - loss: 0.1162 - accuracy: 0.9867 - val_loss: 0.5456 - val_accuracy: 0.7523\n",
            "Epoch 9/10\n",
            "16/16 [==============================] - 7s 465ms/step - loss: 0.0917 - accuracy: 0.9915 - val_loss: 0.5954 - val_accuracy: 0.7523\n",
            "Epoch 10/10\n",
            "16/16 [==============================] - 7s 461ms/step - loss: 0.0686 - accuracy: 0.9915 - val_loss: 0.5948 - val_accuracy: 0.7523\n",
            "1    43\n",
            "0    37\n",
            "Name: AbL, dtype: int64\n",
            "0.8449612403100775\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "254      0.0  1.0\n",
            "541      1.0  0.0\n",
            "389      1.0  0.0\n",
            "96       1.0  0.0\n",
            "160      0.0  1.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 202\n",
            "\t95percentile : 348\n",
            "\t99percentile : 516\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n",
            "\n",
            "\n",
            "begin training using onecycle policy with max lr of 2e-05...\n",
            "Epoch 1/10\n",
            "20/20 [==============================] - 18s 503ms/step - loss: 0.6966 - accuracy: 0.4704 - val_loss: 0.6880 - val_accuracy: 0.6147\n",
            "Epoch 2/10\n",
            "20/20 [==============================] - 9s 434ms/step - loss: 0.6752 - accuracy: 0.7315 - val_loss: 0.6641 - val_accuracy: 0.6055\n",
            "Epoch 3/10\n",
            "20/20 [==============================] - 9s 445ms/step - loss: 0.6643 - accuracy: 0.5987 - val_loss: 0.6383 - val_accuracy: 0.6422\n",
            "Epoch 4/10\n",
            "20/20 [==============================] - 9s 452ms/step - loss: 0.6298 - accuracy: 0.7407 - val_loss: 0.5995 - val_accuracy: 0.7982\n",
            "Epoch 5/10\n",
            "20/20 [==============================] - 9s 454ms/step - loss: 0.5323 - accuracy: 0.8637 - val_loss: 0.4727 - val_accuracy: 0.7890\n",
            "Epoch 6/10\n",
            "20/20 [==============================] - 9s 445ms/step - loss: 0.3219 - accuracy: 0.8870 - val_loss: 0.3987 - val_accuracy: 0.8257\n",
            "Epoch 7/10\n",
            "20/20 [==============================] - 9s 439ms/step - loss: 0.1509 - accuracy: 0.9848 - val_loss: 0.5073 - val_accuracy: 0.7706\n",
            "Epoch 8/10\n",
            "20/20 [==============================] - 9s 435ms/step - loss: 0.0788 - accuracy: 0.9795 - val_loss: 0.4918 - val_accuracy: 0.8349\n",
            "Epoch 9/10\n",
            "20/20 [==============================] - 9s 433ms/step - loss: 0.0420 - accuracy: 1.0000 - val_loss: 0.5656 - val_accuracy: 0.7890\n",
            "Epoch 10/10\n",
            "20/20 [==============================] - 9s 433ms/step - loss: 0.0316 - accuracy: 1.0000 - val_loss: 0.5934 - val_accuracy: 0.7890\n",
            "0    57\n",
            "1    43\n",
            "Name: AbL, dtype: int64\n",
            "0.8819591261451726\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZknIENUfeepz",
        "outputId": "88d5d403-c303-4be9-dd2a-0787b0569878"
      },
      "source": [
        "  print(df_tra.AbL.value_counts())"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0    60\n",
            "1    40\n",
            "Name: AbL, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Hye2nMblDUNM"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UtvGiOu-DUQh"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_YIZd1lEDUTj"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "149-rZxuDUZ7"
      },
      "source": [
        "df_train = df_train.sample(n=10)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 358
        },
        "id": "Ya1t0MesOI_q",
        "outputId": "4abcba7f-7b90-46a7-8dd2-b2c4b278acdd"
      },
      "source": [
        "df_train "
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>AbstractPlusTitle</th>\n",
              "      <th>AbL</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>31</th>\n",
              "      <td>primary dysmenorrhea prevalent condition causi...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>122</th>\n",
              "      <td>examine mindfulness meditation mm was associat...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>290</th>\n",
              "      <td>haematological malignancies malignant neoplasm...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>267</th>\n",
              "      <td>patients progressive kidney disease experience...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>251</th>\n",
              "      <td>research impact mind body training ask partici...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>417</th>\n",
              "      <td>cognitive behavioral therapy insomnia cbt show...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>299</th>\n",
              "      <td>verify clinical efficacy acupuncture moxibusti...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>511</th>\n",
              "      <td>tinnitus perception sound absence acoustic sti...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>521</th>\n",
              "      <td>studies shown aromatherapy improve health prob...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>26</th>\n",
              "      <td>anxiety associated sleep disturbance insomnia ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                     AbstractPlusTitle  AbL\n",
              "31   primary dysmenorrhea prevalent condition causi...    0\n",
              "122  examine mindfulness meditation mm was associat...    0\n",
              "290  haematological malignancies malignant neoplasm...    0\n",
              "267  patients progressive kidney disease experience...    1\n",
              "251  research impact mind body training ask partici...    0\n",
              "417  cognitive behavioral therapy insomnia cbt show...    0\n",
              "299  verify clinical efficacy acupuncture moxibusti...    1\n",
              "511  tinnitus perception sound absence acoustic sti...    0\n",
              "521  studies shown aromatherapy improve health prob...    1\n",
              "26   anxiety associated sleep disturbance insomnia ...    0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 9
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 492
        },
        "id": "4X8Mcj8HVbSd",
        "outputId": "6c71422a-f81b-485b-af76-0e2b4220eec4"
      },
      "source": [
        "(train, val, preproc) = text.texts_from_df(train_df=df_train, text_column='AbstractPlusTitle', label_columns='AbL',\n",
        "                                           val_df = df_valid,\n",
        "                                           maxlen = 512, #length of each doc aligned to 512\n",
        "                                           preprocess_mode = 'distilbert')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "521      0.0  1.0\n",
            "417      1.0  0.0\n",
            "122      1.0  0.0\n",
            "251      1.0  0.0\n",
            "511      1.0  0.0\n",
            "['not_AbL', 'AbL']\n",
            "     not_AbL  AbL\n",
            "176      1.0  0.0\n",
            "77       1.0  0.0\n",
            "361      1.0  0.0\n",
            "90       1.0  0.0\n",
            "368      1.0  0.0\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 195\n",
            "\t95percentile : 306\n",
            "\t99percentile : 367\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 194\n",
            "\t95percentile : 302\n",
            "\t99percentile : 550\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ACyoGnL_1l04"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ca70jYSF5_-I"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5frjEhY_6AAf"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "I1PGWDO7_ifP"
      },
      "source": [
        "#On 87 QA fulltext"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "IVgZYsCi6ADF"
      },
      "source": [
        "dfulQA = pd.read_pickle('/content/drive/MyDrive/HS651_CAMs_systematic_review_automation/FulltextQA Concat/dfmerge87.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bXhRS-c96_dV"
      },
      "source": [
        "#dfulQA.to_excel('/content/drive/MyDrive/HS651_CAMs_systematic_review_automation/FulltextQA Concat/dfmerge87.xlsx')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "B9nczeoX6AFc"
      },
      "source": [
        "#dfulQA['abs_titl_qa'] = dfulQA['AbstractPlusTitle_x']+dfulQA['TreatmentGroupAnswer']+dfulQA['AnxietyMeasureGroupAnswer']+dfulQA['DepressionMeasureGroupAnswer']+dfulQA['SleepMeasureGroupAnswer']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "U19iaDXj-jWh"
      },
      "source": [
        "dfulQA['abs_titl_qa'] = dfulQA['AbstractPlusTitle_x']+dfulQA['TreatmentGroupAnswer']+dfulQA['SleepMeasureGroupAnswer']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Wav-6mnpCbF1"
      },
      "source": [
        "#dfulQA['qas'] = dfulQA['TreatmentGroupAnswer']+dfulQA['AnxietyMeasureGroupAnswer']+dfulQA['DepressionMeasureGroupAnswer']+dfulQA['SleepMeasureGroupAnswer']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KpjYDQgK6AH-"
      },
      "source": [
        "#dfulQA.head()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i9aexkxA8l8p"
      },
      "source": [
        "def preprocess_text(dfulQA):\n",
        "    dfulQA.qas = dfulQA.qas.fillna(' ')\n",
        "    dfulQA.qas = dfulQA.qas.str.replace('\\n',' ')\n",
        "    dfulQA.qas = dfulQA.qas.str.replace('\\r',' ')\n",
        "    return dfulQA"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "LV7r6r7a9qzU"
      },
      "source": [
        "spacy_stopwords = spacy.lang.en.stop_words.STOP_WORDS"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "BGTANjSF9qzZ"
      },
      "source": [
        "nlp = spacy.load('en')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QRnl9dDH9qzZ"
      },
      "source": [
        "#nlp.Defaults.stop_words -= {\"will\",\"had\",\"was\",\"were\",\"did\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3uJrfk0XB5qH"
      },
      "source": [
        "nlp.Defaults.stop_words |= {\"None\",\"vs.\"}"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YAR8bLgKCa0o"
      },
      "source": [
        "#nlp.vocab['None'].is_stop = True"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "HF7G0_D09qza"
      },
      "source": [
        "new_stopwords = spacy_stopwords"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZeQY-VIK8xyP"
      },
      "source": [
        "def clean_text(x):\n",
        "  #x=re.sub(r\"\\'ll\",\" will \",x) ----  In academic paper, people tend not to use contraction\n",
        "  #x=re.sub('[^a-zA-Z0-9]', ' ',x).lower()\n",
        "  x = re.sub('[^a-zA-Z]', ' ',x).lower()\n",
        "  x = x.split()\n",
        "  new_list = []\n",
        "  for word in x:\n",
        "    if word not in new_stopwords: \n",
        "      new_list.append(word)\n",
        "  return \" \".join(new_list)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YfeGDWFV6APY"
      },
      "source": [
        "dfulQA['abs_titl_qa'] = dfulQA.abs_titl_qa.apply(lambda x:clean_text(x))"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "lfwPl7cH-DCs",
        "outputId": "f3be8abd-6ad2-41f1-9d2c-cef7f4dac59a"
      },
      "source": [
        "dfulQA"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Title_x</th>\n",
              "      <th>JournalInfo_x</th>\n",
              "      <th>Abstract_x</th>\n",
              "      <th>TiL_x</th>\n",
              "      <th>AbL_x</th>\n",
              "      <th>AbstractPlusTitle_x</th>\n",
              "      <th>PMID_x</th>\n",
              "      <th>PMCID</th>\n",
              "      <th>TreatmentGroupAnswer</th>\n",
              "      <th>SleepMeasureGroupAnswer</th>\n",
              "      <th>DepressionMeasureGroupAnswer</th>\n",
              "      <th>AnxietyMeasureGroupAnswer</th>\n",
              "      <th>InterviewAnswer</th>\n",
              "      <th>FuL</th>\n",
              "      <th>abs_titl_qa</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>\"Tai Chi Chih Compared With Cognitive Behavior...</td>\n",
              "      <td>J Clin Oncol</td>\n",
              "      <td>Purpose Cognitive behavioral therapy for insom...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Purpose Cognitive behavioral therapy for insom...</td>\n",
              "      <td>28489508</td>\n",
              "      <td>PMC5549450</td>\n",
              "      <td>tcc; cbt-i tcc; tcc v cbt-i</td>\n",
              "      <td>$ 5 points; insomnia severity index; pittsburg...</td>\n",
              "      <td>None; ids-c; ids-c</td>\n",
              "      <td>pre-sleep arousal scale; pre-sleep arousal; se...</td>\n",
              "      <td>months; 6 months 15 months; 3 months 1 month</td>\n",
              "      <td>1.0</td>\n",
              "      <td>purpose cognitive behavioral therapy insomnia ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>\"Cognitive behavioral therapy vs. Tai Chi for ...</td>\n",
              "      <td>Sleep</td>\n",
              "      <td>To investigate the comparative efficacy of cog...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>To investigate the comparative efficacy of cog...</td>\n",
              "      <td>25142571</td>\n",
              "      <td>PMC4153053</td>\n",
              "      <td>None; comparable regard background char- acter...</td>\n",
              "      <td>athens insomnia scale; epworth sleepi- ness; a...</td>\n",
              "      <td>ids-c; clinician rating scale; ids-c</td>\n",
              "      <td>athens insomnia scale; athens insomnia; insomn...</td>\n",
              "      <td>year; None; 16 months</td>\n",
              "      <td>1.0</td>\n",
              "      <td>investigate comparative efficacy cognitive beh...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>\"Tai chi/yoga reduces prenatal depression, anx...</td>\n",
              "      <td>Complement Ther Clin Pract</td>\n",
              "      <td>Ninety-two prenatally depressed pregnant women...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Ninety-two prenatally depressed pregnant women...</td>\n",
              "      <td>23337557</td>\n",
              "      <td>PMC3730281</td>\n",
              "      <td>tai chi/yoga; tai chi/yoga; costly therapy</td>\n",
              "      <td>None; 15-item; 15</td>\n",
              "      <td>ces-d; ces-d; frequency current depressive sym...</td>\n",
              "      <td>anxiety stai; anxiety stai scale; stai</td>\n",
              "      <td>15 min; 3 weeks gestation; 15 min</td>\n",
              "      <td>1.0</td>\n",
              "      <td>ninety prenatally depressed pregnant women ran...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>\"Randomized controlled trial of Qigong/Tai Chi...</td>\n",
              "      <td>Ann Behav Med</td>\n",
              "      <td>Many breast cancer survivors experience fatigu...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Many breast cancer survivors experience fatigu...</td>\n",
              "      <td>25124456</td>\n",
              "      <td>PMC4329282</td>\n",
              "      <td>qg/tce versus sqg; qg/tce versus sqg; None</td>\n",
              "      <td>pittsburgh sleep quality index; None; 60–65%</td>\n",
              "      <td>beck depression inventory pittsburgh sleep qua...</td>\n",
              "      <td>depression anxiety; depression; depression anx...</td>\n",
              "      <td>short long term; 3 month follow-up; 3 months</td>\n",
              "      <td>1.0</td>\n",
              "      <td>breast cancer survivors experience fatigue moo...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>\"Mindfulness meditation and improvement in sle...</td>\n",
              "      <td>JAMA Intern Med</td>\n",
              "      <td>Sleep disturbances are most prevalent among ol...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Sleep disturbances are most prevalent among ol...</td>\n",
              "      <td>25686304</td>\n",
              "      <td>PMC4407465</td>\n",
              "      <td>None; originally randomized conditions; nf-κb</td>\n",
              "      <td>athens insomnia scale; athens insomnia scale; ...</td>\n",
              "      <td>patient health questionnaire 9 score &gt;14; None...</td>\n",
              "      <td>beckdepression inventory ii; beckdepression in...</td>\n",
              "      <td>10 days; 10- week; 6-week</td>\n",
              "      <td>0.0</td>\n",
              "      <td>sleep disturbances prevalent older adults untr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>82</th>\n",
              "      <td>Sound therapy (using amplification devices and...</td>\n",
              "      <td>Cochrane Database Syst Rev</td>\n",
              "      <td>Tinnitus affects 10% to 15% of the adult popul...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Tinnitus affects 10% to 15% of the adult popul...</td>\n",
              "      <td>30589445</td>\n",
              "      <td>PMC6517157</td>\n",
              "      <td>None; combine groups; combination device (n = 15</td>\n",
              "      <td>None; 3; graded 3</td>\n",
              "      <td>hospital anxiety depression scale; hospital an...</td>\n",
              "      <td>depression; depression; tinnitus discomfort</td>\n",
              "      <td>6 months; 6 weeks; 6 weeks</td>\n",
              "      <td>0.0</td>\n",
              "      <td>tinnitus affects adult population experiencing...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>83</th>\n",
              "      <td>Efficacy of \"Pinggan Formula\" in Controlling A...</td>\n",
              "      <td>Evid Based Complement Alternat Med</td>\n",
              "      <td>To explore a new treatment that can proceed fr...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>To explore a new treatment that can proceed fr...</td>\n",
              "      <td>31827555</td>\n",
              "      <td>PMC6885174</td>\n",
              "      <td>control group; control group; randomly :1 ratio</td>\n",
              "      <td>0.95 (0.40 1.50; None; 5.90± 1.40</td>\n",
              "      <td>None; 6.80± .30; 4.04 ( .95 5.1 )</td>\n",
              "      <td>None; 9.60± 1.30 9.00± .10; − 0.65 (− 1.50 0. 1</td>\n",
              "      <td>1 year; 11.5 days; 1 year</td>\n",
              "      <td>0.0</td>\n",
              "      <td>explore new treatment proceed control blood pr...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>84</th>\n",
              "      <td>A Multimodal Stress-Prevention Program Supplem...</td>\n",
              "      <td>Int J Environ Res Public Health</td>\n",
              "      <td>This study compared the effectiveness of a 12-...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>This study compared the effectiveness of a 12-...</td>\n",
              "      <td>33321849</td>\n",
              "      <td>PMC7763316</td>\n",
              "      <td>None; cancer; tc vs. notc</td>\n",
              "      <td>insomnia severity index; None; insomnia severi...</td>\n",
              "      <td>two-item questionnaire; self-employed’s work-r...</td>\n",
              "      <td>gad- m; generalized anxiety disorder; self-emp...</td>\n",
              "      <td>9 months; 3-week; 1 -month</td>\n",
              "      <td>0.0</td>\n",
              "      <td>study compared effectiveness day stress preven...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>85</th>\n",
              "      <td>A parallel randomized controlled trial examini...</td>\n",
              "      <td>PLoS One</td>\n",
              "      <td>This double-blind, two-arm parallel randomized...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>This double-blind, two-arm parallel randomized...</td>\n",
              "      <td>30822311</td>\n",
              "      <td>PMC6396935</td>\n",
              "      <td>gamma range rss vs. complex wave rss; treatmen...</td>\n",
              "      <td>pittsburgh sleep quality index; pittsburgh sle...</td>\n",
              "      <td>None; patient health questionnaire phq-9; 8 (3 %</td>\n",
              "      <td>revised fibromyalgia impact questionnaire; rev...</td>\n",
              "      <td>None; week 1 week 5; week 1 week 5</td>\n",
              "      <td>0.0</td>\n",
              "      <td>double blind arm parallel randomized controlle...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>86</th>\n",
              "      <td>Effects of Manual Therapy on Fatigue, Pain, an...</td>\n",
              "      <td>Int J Environ Res Public Health</td>\n",
              "      <td>Fibromyalgia is a condition characterised by c...</td>\n",
              "      <td>1</td>\n",
              "      <td>1</td>\n",
              "      <td>Fibromyalgia is a condition characterised by c...</td>\n",
              "      <td>32604939</td>\n",
              "      <td>PMC7345776</td>\n",
              "      <td>None; experimental group control group; placebo</td>\n",
              "      <td>pittsburgh; pittsburgh test; pittsburgh test</td>\n",
              "      <td>melancholy; melancholy; melancholy</td>\n",
              "      <td>stress; None; stress</td>\n",
              "      <td>twice week months; None; weeks 0 5</td>\n",
              "      <td>0.0</td>\n",
              "      <td>fibromyalgia condition characterised chronic w...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>87 rows × 15 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "                                              Title_x  ...                                        abs_titl_qa\n",
              "0   \"Tai Chi Chih Compared With Cognitive Behavior...  ...  purpose cognitive behavioral therapy insomnia ...\n",
              "1   \"Cognitive behavioral therapy vs. Tai Chi for ...  ...  investigate comparative efficacy cognitive beh...\n",
              "2   \"Tai chi/yoga reduces prenatal depression, anx...  ...  ninety prenatally depressed pregnant women ran...\n",
              "3   \"Randomized controlled trial of Qigong/Tai Chi...  ...  breast cancer survivors experience fatigue moo...\n",
              "4   \"Mindfulness meditation and improvement in sle...  ...  sleep disturbances prevalent older adults untr...\n",
              "..                                                ...  ...                                                ...\n",
              "82  Sound therapy (using amplification devices and...  ...  tinnitus affects adult population experiencing...\n",
              "83  Efficacy of \"Pinggan Formula\" in Controlling A...  ...  explore new treatment proceed control blood pr...\n",
              "84  A Multimodal Stress-Prevention Program Supplem...  ...  study compared effectiveness day stress preven...\n",
              "85  A parallel randomized controlled trial examini...  ...  double blind arm parallel randomized controlle...\n",
              "86  Effects of Manual Therapy on Fatigue, Pain, an...  ...  fibromyalgia condition characterised chronic w...\n",
              "\n",
              "[87 rows x 15 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DQrHYjn4Duzl"
      },
      "source": [
        "#dfulQA.to_excel('/content/drive/MyDrive/HS651_CAMs_systematic_review_automation/FulltextQA Concat/preproQA.xlsx')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mHEMBb1aMLCA"
      },
      "source": [
        "dfulQA.to_pickle('/content/drive/MyDrive/HS651_CAMs_systematic_review_automation/FulltextQA Concat/preproQA.pkl')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xkl1iKw-6AR9"
      },
      "source": [
        "dful = dfulQA[['abs_titl_qa','FuL']]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "X32nIM9f6AU4"
      },
      "source": [
        "df_valid = dful.sample(frac = 0.2, random_state = 42)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "DCq6v4-q6AW9"
      },
      "source": [
        "df_train = dful.drop(df_valid.index)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "zURH2KYr6AY4",
        "outputId": "c8af2dce-1bc2-40dd-c35a-8154e6ea2a05"
      },
      "source": [
        "len(df_train)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "70"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 30
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        },
        "id": "7GDu_hJl6AbR",
        "outputId": "8c596097-872a-4e2a-f93c-632e6c0ded20"
      },
      "source": [
        "!pip3 install --upgrade tensorflow"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting tensorflow\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/aa/fd/993aa1333eb54d9f000863fe8ec61e41d12eb833dea51484c76c038718b5/tensorflow-2.5.0-cp37-cp37m-manylinux2010_x86_64.whl (454.3MB)\n",
            "\u001b[K     |████████████████████████████████| 454.3MB 36kB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: keras-preprocessing~=1.1.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.2)\n",
            "Requirement already satisfied, skipping upgrade: wheel~=0.35 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.36.2)\n",
            "Collecting tensorflow-estimator<2.6.0,>=2.5.0rc0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ec/78/b27f73e923becc6e79e18fe112cf75e3200d1ee35b0dba8fa46181bce56c/tensorflow_estimator-2.5.0-py2.py3-none-any.whl (462kB)\n",
            "\u001b[K     |████████████████████████████████| 471kB 37.1MB/s \n",
            "\u001b[?25hCollecting grpcio~=1.34.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/d5/d1/f38a91d8724706427fe973a7dfa11e938cee98aa7196b03d870a25a08bab/grpcio-1.34.1-cp37-cp37m-manylinux2014_x86_64.whl (4.0MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0MB 42.7MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: typing-extensions~=3.7.4 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.7.4.3)\n",
            "Requirement already satisfied, skipping upgrade: opt-einsum~=3.3.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.3.0)\n",
            "Collecting tensorboard~=2.5\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/44/f5/7feea02a3fb54d5db827ac4b822a7ba8933826b36de21880518250b8733a/tensorboard-2.5.0-py3-none-any.whl (6.0MB)\n",
            "\u001b[K     |████████████████████████████████| 6.0MB 31.0MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: flatbuffers~=1.12.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.12)\n",
            "Requirement already satisfied, skipping upgrade: six~=1.15.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.15.0)\n",
            "Collecting h5py~=3.1.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/9d/74/9eae2bedd8201ab464308f42c601a12d79727a1c87f0c867fdefb212c6cf/h5py-3.1.0-cp37-cp37m-manylinux1_x86_64.whl (4.0MB)\n",
            "\u001b[K     |████████████████████████████████| 4.0MB 47.4MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: astunparse~=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.6.3)\n",
            "Collecting gast==0.4.0\n",
            "  Downloading https://files.pythonhosted.org/packages/b6/48/583c032b79ae5b3daa02225a675aeb673e58d2cb698e78510feceb11958c/gast-0.4.0-py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: wrapt~=1.12.1 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.12.1)\n",
            "Requirement already satisfied, skipping upgrade: protobuf>=3.9.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (3.12.4)\n",
            "Collecting keras-nightly~=2.5.0.dev\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/aa/e7/53bc896aa4e11a87aac10a625c676b3a3d57d1c8d9929e4809d31fa0b7d5/keras_nightly-2.5.0.dev2021032900-py2.py3-none-any.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 43.1MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: google-pasta~=0.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.2.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy~=1.19.2 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.19.5)\n",
            "Requirement already satisfied, skipping upgrade: termcolor~=1.1.0 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (1.1.0)\n",
            "Requirement already satisfied, skipping upgrade: absl-py~=0.10 in /usr/local/lib/python3.7/dist-packages (from tensorflow) (0.12.0)\n",
            "Requirement already satisfied, skipping upgrade: markdown>=2.6.8 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow) (3.3.4)\n",
            "Requirement already satisfied, skipping upgrade: requests<3,>=2.21.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow) (2.23.0)\n",
            "Requirement already satisfied, skipping upgrade: setuptools>=41.0.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow) (56.1.0)\n",
            "Requirement already satisfied, skipping upgrade: werkzeug>=0.11.15 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow) (1.0.1)\n",
            "Requirement already satisfied, skipping upgrade: google-auth<2,>=1.6.3 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow) (1.30.0)\n",
            "Requirement already satisfied, skipping upgrade: google-auth-oauthlib<0.5,>=0.4.1 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow) (0.4.4)\n",
            "Requirement already satisfied, skipping upgrade: tensorboard-plugin-wit>=1.6.0 in /usr/local/lib/python3.7/dist-packages (from tensorboard~=2.5->tensorflow) (1.8.0)\n",
            "Collecting tensorboard-data-server<0.7.0,>=0.6.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/60/f9/802efd84988bffd9f644c03b6e66fde8e76c3aa33db4279ddd11c5d61f4b/tensorboard_data_server-0.6.1-py3-none-manylinux2010_x86_64.whl (4.9MB)\n",
            "\u001b[K     |████████████████████████████████| 4.9MB 42.4MB/s \n",
            "\u001b[?25hCollecting cached-property; python_version < \"3.8\"\n",
            "  Downloading https://files.pythonhosted.org/packages/48/19/f2090f7dad41e225c7f2326e4cfe6fff49e57dedb5b53636c9551f86b069/cached_property-1.5.2-py2.py3-none-any.whl\n",
            "Requirement already satisfied, skipping upgrade: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from markdown>=2.6.8->tensorboard~=2.5->tensorflow) (4.0.1)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow) (2020.12.5)\n",
            "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow) (2.10)\n",
            "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests<3,>=2.21.0->tensorboard~=2.5->tensorflow) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow) (0.2.8)\n",
            "Requirement already satisfied, skipping upgrade: cachetools<5.0,>=2.0.0 in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow) (4.2.2)\n",
            "Requirement already satisfied, skipping upgrade: rsa<5,>=3.1.4; python_version >= \"3.6\" in /usr/local/lib/python3.7/dist-packages (from google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow) (4.7.2)\n",
            "Requirement already satisfied, skipping upgrade: requests-oauthlib>=0.7.0 in /usr/local/lib/python3.7/dist-packages (from google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow) (1.3.0)\n",
            "Requirement already satisfied, skipping upgrade: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->markdown>=2.6.8->tensorboard~=2.5->tensorflow) (3.4.1)\n",
            "Requirement already satisfied, skipping upgrade: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.7/dist-packages (from pyasn1-modules>=0.2.1->google-auth<2,>=1.6.3->tensorboard~=2.5->tensorflow) (0.4.8)\n",
            "Requirement already satisfied, skipping upgrade: oauthlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from requests-oauthlib>=0.7.0->google-auth-oauthlib<0.5,>=0.4.1->tensorboard~=2.5->tensorflow) (3.1.0)\n",
            "Installing collected packages: tensorflow-estimator, grpcio, tensorboard-data-server, tensorboard, cached-property, h5py, gast, keras-nightly, tensorflow\n",
            "  Found existing installation: tensorflow-estimator 2.4.0\n",
            "    Uninstalling tensorflow-estimator-2.4.0:\n",
            "      Successfully uninstalled tensorflow-estimator-2.4.0\n",
            "  Found existing installation: grpcio 1.32.0\n",
            "    Uninstalling grpcio-1.32.0:\n",
            "      Successfully uninstalled grpcio-1.32.0\n",
            "  Found existing installation: tensorboard 2.4.1\n",
            "    Uninstalling tensorboard-2.4.1:\n",
            "      Successfully uninstalled tensorboard-2.4.1\n",
            "  Found existing installation: h5py 2.10.0\n",
            "    Uninstalling h5py-2.10.0:\n",
            "      Successfully uninstalled h5py-2.10.0\n",
            "  Found existing installation: gast 0.3.3\n",
            "    Uninstalling gast-0.3.3:\n",
            "      Successfully uninstalled gast-0.3.3\n",
            "  Found existing installation: tensorflow 2.4.1\n",
            "    Uninstalling tensorflow-2.4.1:\n",
            "      Successfully uninstalled tensorflow-2.4.1\n",
            "Successfully installed cached-property-1.5.2 gast-0.4.0 grpcio-1.34.1 h5py-3.1.0 keras-nightly-2.5.0.dev2021032900 tensorboard-2.5.0 tensorboard-data-server-0.6.1 tensorflow-2.5.0 tensorflow-estimator-2.5.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.colab-display-data+json": {
              "pip_warning": {
                "packages": [
                  "grpc"
                ]
              }
            }
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7zlF-5n96Ad8",
        "outputId": "4374c53e-54ee-41a7-c3ff-c5d17360c75e"
      },
      "source": [
        "!pip install ktrain"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting ktrain\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/99/67/31cab9d7c0e23333aebc28b082659c1528f9ab7e22d00e7237efe4fc14f6/ktrain-0.26.2.tar.gz (25.3MB)\n",
            "\u001b[K     |████████████████████████████████| 25.3MB 114kB/s \n",
            "\u001b[?25hCollecting scikit-learn==0.23.2\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f4/cb/64623369f348e9bfb29ff898a57ac7c91ed4921f228e9726546614d63ccb/scikit_learn-0.23.2-cp37-cp37m-manylinux1_x86_64.whl (6.8MB)\n",
            "\u001b[K     |████████████████████████████████| 6.8MB 42.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: matplotlib>=3.0.0 in /usr/local/lib/python3.7/dist-packages (from ktrain) (3.2.2)\n",
            "Requirement already satisfied: pandas>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.1.5)\n",
            "Requirement already satisfied: fastprogress>=0.1.21 in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.0.0)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from ktrain) (2.23.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from ktrain) (1.0.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from ktrain) (20.9)\n",
            "Requirement already satisfied: ipython in /usr/local/lib/python3.7/dist-packages (from ktrain) (5.5.0)\n",
            "Collecting langdetect\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0e/72/a3add0e4eec4eb9e2569554f7c70f4a3c27712f40e3284d483e88094cc0e/langdetect-1.0.9.tar.gz (981kB)\n",
            "\u001b[K     |████████████████████████████████| 983kB 44.8MB/s \n",
            "\u001b[?25hRequirement already satisfied: jieba in /usr/local/lib/python3.7/dist-packages (from ktrain) (0.42.1)\n",
            "Collecting cchardet\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/80/72/a4fba7559978de00cf44081c548c5d294bf00ac7dcda2db405d2baa8c67a/cchardet-2.1.7-cp37-cp37m-manylinux2010_x86_64.whl (263kB)\n",
            "\u001b[K     |████████████████████████████████| 266kB 49.3MB/s \n",
            "\u001b[?25hCollecting syntok\n",
            "  Downloading https://files.pythonhosted.org/packages/8c/76/a49e73a04b3e3a14ce232e8e28a1587f8108baa665644fe8c40e307e792e/syntok-1.3.1.tar.gz\n",
            "Collecting seqeval==0.0.19\n",
            "  Downloading https://files.pythonhosted.org/packages/93/e5/b7705156a77f742cfe4fc6f22d0c71591edb2d243328dff2f8fc0f933ab6/seqeval-0.0.19.tar.gz\n",
            "Collecting transformers<=4.3.3,>=4.0.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/54/5ca07ec9569d2f232f3166de5457b63943882f7950ddfcc887732fc7fb23/transformers-4.3.3-py3-none-any.whl (1.9MB)\n",
            "\u001b[K     |████████████████████████████████| 1.9MB 32.3MB/s \n",
            "\u001b[?25hCollecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f5/99/e0808cb947ba10f575839c43e8fafc9cc44e4a7a2c8f79c60db48220a577/sentencepiece-0.1.95-cp37-cp37m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\u001b[K     |████████████████████████████████| 1.2MB 38.9MB/s \n",
            "\u001b[?25hCollecting keras_bert>=0.86.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e2/7f/95fabd29f4502924fa3f09ff6538c5a7d290dfef2c2fe076d3d1a16e08f0/keras-bert-0.86.0.tar.gz\n",
            "Requirement already satisfied: networkx>=2.3 in /usr/local/lib/python3.7/dist-packages (from ktrain) (2.5.1)\n",
            "Collecting whoosh\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ba/19/24d0f1f454a2c1eb689ca28d2f178db81e5024f42d82729a4ff6771155cf/Whoosh-2.7.4-py2.py3-none-any.whl (468kB)\n",
            "\u001b[K     |████████████████████████████████| 471kB 42.9MB/s \n",
            "\u001b[?25hRequirement already satisfied: numpy>=1.13.3 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.2->ktrain) (1.19.5)\n",
            "Collecting threadpoolctl>=2.0.0\n",
            "  Downloading https://files.pythonhosted.org/packages/f7/12/ec3f2e203afa394a149911729357aa48affc59c20e2c1c8297a60f33f133/threadpoolctl-2.1.0-py3-none-any.whl\n",
            "Requirement already satisfied: scipy>=0.19.1 in /usr/local/lib/python3.7/dist-packages (from scikit-learn==0.23.2->ktrain) (1.4.1)\n",
            "Requirement already satisfied: cycler>=0.10 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (0.10.0)\n",
            "Requirement already satisfied: pyparsing!=2.0.4,!=2.1.2,!=2.1.6,>=2.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (2.4.7)\n",
            "Requirement already satisfied: python-dateutil>=2.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (2.8.1)\n",
            "Requirement already satisfied: kiwisolver>=1.0.1 in /usr/local/lib/python3.7/dist-packages (from matplotlib>=3.0.0->ktrain) (1.3.1)\n",
            "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=1.0.1->ktrain) (2018.9)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (3.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->ktrain) (2020.12.5)\n",
            "Requirement already satisfied: pickleshare in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (0.7.5)\n",
            "Requirement already satisfied: decorator in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (4.4.2)\n",
            "Requirement already satisfied: simplegeneric>0.8 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (0.8.1)\n",
            "Requirement already satisfied: pygments in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (2.6.1)\n",
            "Requirement already satisfied: prompt-toolkit<2.0.0,>=1.0.4 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (1.0.18)\n",
            "Requirement already satisfied: pexpect; sys_platform != \"win32\" in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (4.8.0)\n",
            "Requirement already satisfied: setuptools>=18.5 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (56.1.0)\n",
            "Requirement already satisfied: traitlets>=4.2 in /usr/local/lib/python3.7/dist-packages (from ipython->ktrain) (5.0.5)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from langdetect->ktrain) (1.15.0)\n",
            "Requirement already satisfied: regex in /usr/local/lib/python3.7/dist-packages (from syntok->ktrain) (2019.12.20)\n",
            "Requirement already satisfied: Keras>=2.2.4 in /usr/local/lib/python3.7/dist-packages (from seqeval==0.0.19->ktrain) (2.4.3)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (3.0.12)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (4.41.1)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers<=4.3.3,>=4.0.0->ktrain) (4.0.1)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ae/04/5b870f26a858552025a62f1649c20d29d2672c02ff3c3fb4c688ca46467a/tokenizers-0.10.2-cp37-cp37m-manylinux2010_x86_64.whl (3.3MB)\n",
            "\u001b[K     |████████████████████████████████| 3.3MB 39.3MB/s \n",
            "\u001b[?25hCollecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/75/ee/67241dc87f266093c533a2d4d3d69438e57d7a90abb216fa076e7d475d4a/sacremoses-0.0.45-py3-none-any.whl (895kB)\n",
            "\u001b[K     |████████████████████████████████| 901kB 30.6MB/s \n",
            "\u001b[?25hCollecting keras-transformer>=0.38.0\n",
            "  Downloading https://files.pythonhosted.org/packages/89/6c/d6f0c164f4cc16fbc0d0fea85f5526e87a7d2df7b077809e422a7e626150/keras-transformer-0.38.0.tar.gz\n",
            "Requirement already satisfied: wcwidth in /usr/local/lib/python3.7/dist-packages (from prompt-toolkit<2.0.0,>=1.0.4->ipython->ktrain) (0.2.5)\n",
            "Requirement already satisfied: ptyprocess>=0.5 in /usr/local/lib/python3.7/dist-packages (from pexpect; sys_platform != \"win32\"->ipython->ktrain) (0.7.0)\n",
            "Requirement already satisfied: ipython-genutils in /usr/local/lib/python3.7/dist-packages (from traitlets>=4.2->ipython->ktrain) (0.2.0)\n",
            "Requirement already satisfied: pyyaml in /usr/local/lib/python3.7/dist-packages (from Keras>=2.2.4->seqeval==0.0.19->ktrain) (3.13)\n",
            "Requirement already satisfied: h5py in /usr/local/lib/python3.7/dist-packages (from Keras>=2.2.4->seqeval==0.0.19->ktrain) (3.1.0)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<=4.3.3,>=4.0.0->ktrain) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers<=4.3.3,>=4.0.0->ktrain) (3.4.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers<=4.3.3,>=4.0.0->ktrain) (7.1.2)\n",
            "Collecting keras-pos-embd>=0.11.0\n",
            "  Downloading https://files.pythonhosted.org/packages/09/70/b63ed8fc660da2bb6ae29b9895401c628da5740c048c190b5d7107cadd02/keras-pos-embd-0.11.0.tar.gz\n",
            "Collecting keras-multi-head>=0.27.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e6/32/45adf2549450aca7867deccfa04af80a0ab1ca139af44b16bc669e0e09cd/keras-multi-head-0.27.0.tar.gz\n",
            "Collecting keras-layer-normalization>=0.14.0\n",
            "  Downloading https://files.pythonhosted.org/packages/a4/0e/d1078df0494bac9ce1a67954e5380b6e7569668f0f3b50a9531c62c1fc4a/keras-layer-normalization-0.14.0.tar.gz\n",
            "Collecting keras-position-wise-feed-forward>=0.6.0\n",
            "  Downloading https://files.pythonhosted.org/packages/e3/59/f0faa1037c033059e7e9e7758e6c23b4d1c0772cd48de14c4b6fd4033ad5/keras-position-wise-feed-forward-0.6.0.tar.gz\n",
            "Collecting keras-embed-sim>=0.8.0\n",
            "  Downloading https://files.pythonhosted.org/packages/57/ef/61a1e39082c9e1834a2d09261d4a0b69f7c818b359216d4e1912b20b1c86/keras-embed-sim-0.8.0.tar.gz\n",
            "Requirement already satisfied: cached-property; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from h5py->Keras>=2.2.4->seqeval==0.0.19->ktrain) (1.5.2)\n",
            "Collecting keras-self-attention==0.46.0\n",
            "  Downloading https://files.pythonhosted.org/packages/15/6b/c804924a056955fa1f3ff767945187103cfc851ba9bd0fc5a6c6bc18e2eb/keras-self-attention-0.46.0.tar.gz\n",
            "Building wheels for collected packages: ktrain, langdetect, syntok, seqeval, keras-bert, keras-transformer, keras-pos-embd, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-self-attention\n",
            "  Building wheel for ktrain (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for ktrain: filename=ktrain-0.26.2-cp37-none-any.whl size=25277794 sha256=ddc17ab30da38a1cbd41e2f1c9b564d2d28d32406e679561318264700fbf4137\n",
            "  Stored in directory: /root/.cache/pip/wheels/67/2e/f1/c72afa08df8b2d984b910dea228902ce81dae4511afe9fafd2\n",
            "  Building wheel for langdetect (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for langdetect: filename=langdetect-1.0.9-cp37-none-any.whl size=993223 sha256=cd4ee85c20b611066f6d8045bea3d92f85ad732d255a115c042386256258243a\n",
            "  Stored in directory: /root/.cache/pip/wheels/7e/18/13/038c34057808931c7ddc6c92d3aa015cf1a498df5a70268996\n",
            "  Building wheel for syntok (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for syntok: filename=syntok-1.3.1-cp37-none-any.whl size=20919 sha256=1bee3744de9d6b276b706f53c7bc78b2fcd075de4c5ff4da0a7fd1855202cc28\n",
            "  Stored in directory: /root/.cache/pip/wheels/51/c6/a4/be1920586c49469846bcd2888200bdecfe109ec421dab9be2d\n",
            "  Building wheel for seqeval (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for seqeval: filename=seqeval-0.0.19-cp37-none-any.whl size=9919 sha256=a8d4f86757d1f002de0626904fce61d9c22d6ff143b3f6618e8000a6b0a31921\n",
            "  Stored in directory: /root/.cache/pip/wheels/8d/1f/bf/1198beceed805a2099060975f6281d1b01046dd279e19c97be\n",
            "  Building wheel for keras-bert (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-bert: filename=keras_bert-0.86.0-cp37-none-any.whl size=34144 sha256=bd2cdb48b362903e3ac3e587e92a8c78bd1395c0f8a2d6fe27aea93e4c17ddba\n",
            "  Stored in directory: /root/.cache/pip/wheels/66/f0/b1/748128b58562fc9e31b907bb5e2ab6a35eb37695e83911236b\n",
            "  Building wheel for keras-transformer (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-transformer: filename=keras_transformer-0.38.0-cp37-none-any.whl size=12942 sha256=59eb4dd3a3ad9c5f87a91f303437f35041fa8a70f9b178af58443ac0a9a82f15\n",
            "  Stored in directory: /root/.cache/pip/wheels/e5/fb/3a/37b2b9326c799aa010ae46a04ddb04f320d8c77c0b7e837f4e\n",
            "  Building wheel for keras-pos-embd (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-pos-embd: filename=keras_pos_embd-0.11.0-cp37-none-any.whl size=7554 sha256=1763bd6bb3843eb71beddb27b32639415507dab92b44a16265054b2e3076f666\n",
            "  Stored in directory: /root/.cache/pip/wheels/5b/a1/a0/ce6b1d49ba1a9a76f592e70cf297b05c96bc9f418146761032\n",
            "  Building wheel for keras-multi-head (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-multi-head: filename=keras_multi_head-0.27.0-cp37-none-any.whl size=15611 sha256=c5a3adb7cf5fc95743135d133f97009e58c96145cb2cf21d6b64cc5aa9964281\n",
            "  Stored in directory: /root/.cache/pip/wheels/b5/b4/49/0a0c27dcb93c13af02fea254ff51d1a43a924dd4e5b7a7164d\n",
            "  Building wheel for keras-layer-normalization (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-layer-normalization: filename=keras_layer_normalization-0.14.0-cp37-none-any.whl size=5269 sha256=3a53dbafb76e1a330cf86c6f4aed5192f1992ae2434d5604fff921e6ae3f79eb\n",
            "  Stored in directory: /root/.cache/pip/wheels/54/80/22/a638a7d406fd155e507aa33d703e3fa2612b9eb7bb4f4fe667\n",
            "  Building wheel for keras-position-wise-feed-forward (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-position-wise-feed-forward: filename=keras_position_wise_feed_forward-0.6.0-cp37-none-any.whl size=5623 sha256=087584c9a7ffd912a210035e45bdc5390e3c7a2f2998bf1314ab43f9dcfc6040\n",
            "  Stored in directory: /root/.cache/pip/wheels/39/e2/e2/3514fef126a00574b13bc0b9e23891800158df3a3c19c96e3b\n",
            "  Building wheel for keras-embed-sim (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-embed-sim: filename=keras_embed_sim-0.8.0-cp37-none-any.whl size=4558 sha256=1f80f4b6ee92031705804595beecd792d5bbb38f87a21c950a2ab4fcc7354180\n",
            "  Stored in directory: /root/.cache/pip/wheels/49/45/8b/c111f6cc8bec253e984677de73a6f4f5d2f1649f42aac191c8\n",
            "  Building wheel for keras-self-attention (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for keras-self-attention: filename=keras_self_attention-0.46.0-cp37-none-any.whl size=17278 sha256=4fc9b0335203eb66adbbc2b7a9d9919a34162a36bd43ccad50eca899819dece8\n",
            "  Stored in directory: /root/.cache/pip/wheels/d2/2e/80/fec4c05eb23c8e13b790e26d207d6e0ffe8013fad8c6bdd4d2\n",
            "Successfully built ktrain langdetect syntok seqeval keras-bert keras-transformer keras-pos-embd keras-multi-head keras-layer-normalization keras-position-wise-feed-forward keras-embed-sim keras-self-attention\n",
            "Installing collected packages: threadpoolctl, scikit-learn, langdetect, cchardet, syntok, seqeval, tokenizers, sacremoses, transformers, sentencepiece, keras-pos-embd, keras-self-attention, keras-multi-head, keras-layer-normalization, keras-position-wise-feed-forward, keras-embed-sim, keras-transformer, keras-bert, whoosh, ktrain\n",
            "  Found existing installation: scikit-learn 0.22.2.post1\n",
            "    Uninstalling scikit-learn-0.22.2.post1:\n",
            "      Successfully uninstalled scikit-learn-0.22.2.post1\n",
            "Successfully installed cchardet-2.1.7 keras-bert-0.86.0 keras-embed-sim-0.8.0 keras-layer-normalization-0.14.0 keras-multi-head-0.27.0 keras-pos-embd-0.11.0 keras-position-wise-feed-forward-0.6.0 keras-self-attention-0.46.0 keras-transformer-0.38.0 ktrain-0.26.2 langdetect-1.0.9 sacremoses-0.0.45 scikit-learn-0.23.2 sentencepiece-0.1.95 seqeval-0.0.19 syntok-1.3.1 threadpoolctl-2.1.0 tokenizers-0.10.2 transformers-4.3.3 whoosh-2.7.4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "2PSxQAUA6Afv"
      },
      "source": [
        "import tensorflow as tf\n",
        "from sklearn import metrics\n",
        "import ktrain\n",
        "from ktrain import text"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 731,
          "referenced_widgets": [
            "1f8ad812dbc84d6f8f718c898b166b40",
            "979b2efae6b7483883c52f55723d37db",
            "05d85b9bc1714fbaadc6ea04246a12e9",
            "91c712547d984a40b53026d5e0bf2939",
            "4c7d0aee24e64fc7a9baa70a9468ba1f",
            "925a34155673496fae1240c93f2290b6",
            "8eae09d2a2a2486e8ca9b229cc88cf41",
            "b47a546b84da4b0e81b873058c1e8bb1",
            "a1c8774ad45d421fa644536bf3e3dbff",
            "4df33d4f07d447f5802be99b3d254bdd",
            "908f1629c7714f9eaa00fd20f97f7c47",
            "0ced40c126b648e694e95270935d2364",
            "435c97b291d948539c173c7adb359d63",
            "69cca8431534429ba5d6d1511ebf036d",
            "776a98a9d85e47dd9187a733ec7d6425",
            "d590b6a4cdbe4191ac4c1a3725798344",
            "e3b588ef2143444bb7a7900200abd5db",
            "9fd5c3faecc5455b84d06a5d8ca501a4",
            "668f5c67be1841fbbac6bcc7785bde38",
            "b4f040bc24db40a3ac47c816bca846eb",
            "5fc80178e3ad4b028c090ffa7b8626c5",
            "dc9a6b1b8cbc43b3892d68ccf5c5b89a",
            "2f5220e3e6954b3388f31c6ecd94229e",
            "3ac586698fbf4bd78e5ac15bf1e72157"
          ]
        },
        "id": "xJd-8Xlj6AiZ",
        "outputId": "cdbbcb02-431e-4d83-d83c-a54332c4dcec"
      },
      "source": [
        "(train, val, preproc) = text.texts_from_df(train_df=df_train, \n",
        "                                               text_column='abs_titl_qa', \n",
        "                                               label_columns='FuL',\n",
        "                                               val_df = df_valid,\n",
        "                                               maxlen = 512, #length of each doc aligned to 512\n",
        "                                               preprocess_mode = 'distilbert')"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.7/dist-packages/ktrain/utils.py:605: UserWarning: class_names implies classification but targets array contains float(s) instead of integers or strings\n",
            "  warnings.warn('class_names implies classification but targets array contains float(s) instead of integers or strings')\n",
            "/usr/local/lib/python3.7/dist-packages/ktrain/utils.py:605: UserWarning: class_names implies classification but targets array contains float(s) instead of integers or strings\n",
            "  warnings.warn('class_names implies classification but targets array contains float(s) instead of integers or strings')\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "['not_FuL', 'FuL']\n",
            "   not_FuL  FuL\n",
            "1      0.0  1.0\n",
            "2      0.0  1.0\n",
            "3      0.0  1.0\n",
            "5      1.0  0.0\n",
            "6      1.0  0.0\n",
            "['not_FuL', 'FuL']\n",
            "    not_FuL  FuL\n",
            "76      0.0  1.0\n",
            "0       0.0  1.0\n",
            "26      1.0  0.0\n",
            "22      0.0  1.0\n",
            "12      0.0  1.0\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "1f8ad812dbc84d6f8f718c898b166b40",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=442.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n",
            "preprocessing train...\n",
            "language: en\n",
            "train sequence lengths:\n",
            "\tmean : 186\n",
            "\t95percentile : 247\n",
            "\t99percentile : 357\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "a1c8774ad45d421fa644536bf3e3dbff",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=231508.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "e3b588ef2143444bb7a7900200abd5db",
              "version_minor": 0,
              "version_major": 2
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=466062.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "preprocessing test...\n",
            "language: en\n",
            "test sequence lengths:\n",
            "\tmean : 191\n",
            "\t95percentile : 239\n",
            "\t99percentile : 248\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              ""
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "T0FDQJ3OVbSe",
        "outputId": "1e72697b-bd13-435b-a950-0685c8360526"
      },
      "source": [
        "bertmodel = text.text_classifier(name = 'distilbert', train_data = train, preproc=preproc)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Is Multi-Label? False\n",
            "maxlen is 512\n",
            "done.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lZjHsm73VbSe"
      },
      "source": [
        "learner = ktrain.get_learner(model = bertmodel,\n",
        "                             train_data = train,\n",
        "                             val_data = val,\n",
        "                             batch_size = 6)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 332
        },
        "id": "ht4EN-cHVbSe",
        "outputId": "e74e5232-5c37-489e-fd51-1d5d6c85214e"
      },
      "source": [
        "learner.fit_onecycle(lr = 2e-5, epochs=10)"
      ],
      "execution_count": null,
      "outputs": [
        {
          "output_type": "error",
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-41-e2abd4d7c278>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mlearner\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfit_onecycle\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mlr\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m2e-5\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mepochs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m10\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ktrain/core.py\u001b[0m in \u001b[0;36mfit_onecycle\u001b[0;34m(self, lr, epochs, checkpoint_folder, cycle_momentum, max_momentum, min_momentum, class_weight, callbacks, steps_per_epoch, verbose)\u001b[0m\n\u001b[1;32m    858\u001b[0m                         \u001b[0mcheckpoint_folder\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mcheckpoint_folder\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    859\u001b[0m                         \u001b[0mverbose\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mverbose\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcallbacks\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mkcallbacks\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 860\u001b[0;31m                         steps_per_epoch=steps_per_epoch)\n\u001b[0m\u001b[1;32m    861\u001b[0m         \u001b[0mhist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    862\u001b[0m         \u001b[0mhist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'iterations'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'iterations'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/ktrain/core.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, lr, n_cycles, cycle_len, cycle_mult, lr_decay, checkpoint_folder, early_stopping, class_weight, callbacks, steps_per_epoch, verbose)\u001b[0m\n\u001b[1;32m   1325\u001b[0m                                         \u001b[0mshuffle\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1326\u001b[0m                                         \u001b[0mclass_weight\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mclass_weight\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1327\u001b[0;31m                                         callbacks=kcallbacks)\n\u001b[0m\u001b[1;32m   1328\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0msgdr\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mhist\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msgdr\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'lr'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1329\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhistory\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mhist\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/keras/engine/training.py\u001b[0m in \u001b[0;36mfit\u001b[0;34m(self, x, y, batch_size, epochs, verbose, callbacks, validation_split, validation_data, shuffle, class_weight, sample_weight, initial_epoch, steps_per_epoch, validation_steps, validation_batch_size, validation_freq, max_queue_size, workers, use_multiprocessing)\u001b[0m\n\u001b[1;32m   1181\u001b[0m                 _r=1):\n\u001b[1;32m   1182\u001b[0m               \u001b[0mcallbacks\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mon_train_batch_begin\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstep\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1183\u001b[0;31m               \u001b[0mtmp_logs\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtrain_function\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0miterator\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1184\u001b[0m               \u001b[0;32mif\u001b[0m \u001b[0mdata_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshould_sync\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1185\u001b[0m                 \u001b[0mcontext\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0masync_wait\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    887\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    888\u001b[0m       \u001b[0;32mwith\u001b[0m \u001b[0mOptionalXlaContext\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jit_compile\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 889\u001b[0;31m         \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_call\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    891\u001b[0m       \u001b[0mnew_tracing_count\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexperimental_get_tracing_count\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/def_function.py\u001b[0m in \u001b[0;36m_call\u001b[0;34m(self, *args, **kwds)\u001b[0m\n\u001b[1;32m    915\u001b[0m       \u001b[0;31m# In this case we have created variables on the first call, so we run the\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    916\u001b[0m       \u001b[0;31m# defunned version which is guaranteed to never create variables.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 917\u001b[0;31m       \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateless_fn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m  \u001b[0;31m# pylint: disable=not-callable\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    918\u001b[0m     \u001b[0;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_stateful_fn\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    919\u001b[0m       \u001b[0;31m# Release the lock early so that multiple threads can perform the call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   3022\u001b[0m        filtered_flat_args) = self._maybe_define_function(args, kwargs)\n\u001b[1;32m   3023\u001b[0m     return graph_function._call_flat(\n\u001b[0;32m-> 3024\u001b[0;31m         filtered_flat_args, captured_inputs=graph_function.captured_inputs)  # pylint: disable=protected-access\n\u001b[0m\u001b[1;32m   3025\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   3026\u001b[0m   \u001b[0;34m@\u001b[0m\u001b[0mproperty\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36m_call_flat\u001b[0;34m(self, args, captured_inputs, cancellation_manager)\u001b[0m\n\u001b[1;32m   1959\u001b[0m       \u001b[0;31m# No tape is watching; skip to running the function.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1960\u001b[0m       return self._build_call_outputs(self._inference_function.call(\n\u001b[0;32m-> 1961\u001b[0;31m           ctx, args, cancellation_manager=cancellation_manager))\n\u001b[0m\u001b[1;32m   1962\u001b[0m     forward_backward = self._select_forward_and_backward_functions(\n\u001b[1;32m   1963\u001b[0m         \u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/function.py\u001b[0m in \u001b[0;36mcall\u001b[0;34m(self, ctx, args, cancellation_manager)\u001b[0m\n\u001b[1;32m    594\u001b[0m               \u001b[0minputs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0margs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    595\u001b[0m               \u001b[0mattrs\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mattrs\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 596\u001b[0;31m               ctx=ctx)\n\u001b[0m\u001b[1;32m    597\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    598\u001b[0m           outputs = execute.execute_with_cancellation(\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/tensorflow/python/eager/execute.py\u001b[0m in \u001b[0;36mquick_execute\u001b[0;34m(op_name, num_outputs, inputs, attrs, ctx, name)\u001b[0m\n\u001b[1;32m     58\u001b[0m     \u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mensure_initialized\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     59\u001b[0m     tensors = pywrap_tfe.TFE_Py_Execute(ctx._handle, device_name, op_name,\n\u001b[0;32m---> 60\u001b[0;31m                                         inputs, attrs, num_outputs)\n\u001b[0m\u001b[1;32m     61\u001b[0m   \u001b[0;32mexcept\u001b[0m \u001b[0mcore\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_NotOkStatusException\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mname\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "4td7RyiP6BvF"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "H7NcE2qFVbSf"
      },
      "source": [
        "predictor = ktrain.get_predictor(learner.model, preproc)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "So2SeTkAeAmu"
      },
      "source": [
        "#predictor.save('/content/drive/MyDrive/DistillBert_TitlePlusAbstract')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YeMcfPDSVbSg"
      },
      "source": [
        "texts=df_valid['abs_titl_qa']\n",
        "data=list(texts)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "6Cw1B6ZKVbSg"
      },
      "source": [
        "#data"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oocsRf5YVbSg"
      },
      "source": [
        "###predictor.get_classes()"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "POn-GqzIVbSh"
      },
      "source": [
        "y_pred = predictor.predict(data, return_proba=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "hH3XQRx3VbSh"
      },
      "source": [
        "df_valid.FuL = df_valid.FuL.astype('int')\n",
        "df_train.FuL = df_train.FuL.astype('int')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "mtS9K_GWVbSi"
      },
      "source": [
        "y_test = df_valid['FuL']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "yvxqIa9tVbSi"
      },
      "source": [
        "fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred[:,1])\n",
        "auc = metrics.auc(fpr, tpr)\n",
        "#plt.plot(fpr, tpr)\n",
        "auc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wNONvzE_VbSj"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "1zCRNyqYVbSk"
      },
      "source": [
        "#### Alternate metric reports ####"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GsYUrLt0VbSk"
      },
      "source": [
        "X_test = df_valid.loc [:,'last500']\n",
        "X_test[78]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5CY961diVbSk"
      },
      "source": [
        "X_test = df_valid.loc [:,'last500']\n",
        "\n",
        "# X_test.keys()\n",
        "X_test_BERT = []\n",
        "for keys in X_test.keys():\n",
        "    X_test_BERT.append(X_test[keys])\n",
        "#X_test_BERT[0]"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "jmlAXbHQVbSl"
      },
      "source": [
        "y_pred_BERT = predictor.predict(X_test_BERT, return_proba=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "txfW0b5yVbSl"
      },
      "source": [
        "#predictor.save('/content/drive/MyDrive/BERT_HS651CAMs_pred')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_x9I-mCVi_yL"
      },
      "source": [
        "y_test = df_valid['AbL']"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KO2tPpDpVbSl"
      },
      "source": [
        "y_pred_BERT"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "pMOoeZZOVbSm"
      },
      "source": [
        "fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_BERT[:,1])\n",
        "auc = metrics.auc(fpr, tpr)\n",
        "auc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "uiNB9fwTVbSm"
      },
      "source": [
        "# Load saved model"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "jmbmkpwNVbSm"
      },
      "source": [
        "saved_predictor = ktrain.load_predictor('/content/drive/MyDrive/BERT_HS651CAMs_pred')"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xmg6yeL0VbSm"
      },
      "source": [
        "y_pred_BERT1 = saved_predictor.predict(X_test_BERT, return_proba=True)"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "engYu8WfVbSn"
      },
      "source": [
        "y_pred_BERT1 "
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "collapsed": true,
        "id": "R0t4Mw0oVbSn"
      },
      "source": [
        "fpr, tpr, thresholds = metrics.roc_curve(y_test, y_pred_BERT1[:,1])\n",
        "auc = metrics.auc(fpr, tpr)\n",
        "auc"
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "lJSTT5z-6BtR"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "J4N4krRJVbSo"
      },
      "source": [
        ""
      ],
      "execution_count": null,
      "outputs": []
    }
  ]
}