CLF_BERT copy.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "colab_type": "text",
        "id": "view-in-github"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/zzunebye/Capstone-code-data/blob/main/CLF_BERT.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "v8u0ebtirRci"
      },
      "source": [
        "## Colab"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dQSdijCsmtPH",
        "outputId": "f84e9290-41fe-4a35-ffd0-f7e0e93fa99d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/f9/54/5ca07ec9569d2f232f3166de5457b63943882f7950ddfcc887732fc7fb23/transformers-4.3.3-py3-none-any.whl (1.9MB)\n",
            "\r\u001b[K     |▏                               | 10kB 23.7MB/s eta 0:00:01\r\u001b[K     |▍                               | 20kB 15.8MB/s eta 0:00:01\r\u001b[K     |▌                               | 30kB 13.7MB/s eta 0:00:01\r\u001b[K     |▊                               | 40kB 12.9MB/s eta 0:00:01\r\u001b[K     |▉                               | 51kB 7.7MB/s eta 0:00:01\r\u001b[K     |█                               | 61kB 8.9MB/s eta 0:00:01\r\u001b[K     |█▏                              | 71kB 9.3MB/s eta 0:00:01\r\u001b[K     |█▍                              | 81kB 9.5MB/s eta 0:00:01\r\u001b[K     |█▌                              | 92kB 9.3MB/s eta 0:00:01\r\u001b[K     |█▊                              | 102kB 8.6MB/s eta 0:00:01\r\u001b[K     |██                              | 112kB 8.6MB/s eta 0:00:01\r\u001b[K     |██                              | 122kB 8.6MB/s eta 0:00:01\r\u001b[K     |██▎                             | 133kB 8.6MB/s eta 0:00:01\r\u001b[K     |██▍                             | 143kB 8.6MB/s eta 0:00:01\r\u001b[K     |██▋                             | 153kB 8.6MB/s eta 0:00:01\r\u001b[K     |██▊                             | 163kB 8.6MB/s eta 0:00:01\r\u001b[K     |███                             | 174kB 8.6MB/s eta 0:00:01\r\u001b[K     |███                             | 184kB 8.6MB/s eta 0:00:01\r\u001b[K     |███▎                            | 194kB 8.6MB/s eta 0:00:01\r\u001b[K     |███▌                            | 204kB 8.6MB/s eta 0:00:01\r\u001b[K     |███▋                            | 215kB 8.6MB/s eta 0:00:01\r\u001b[K     |███▉                            | 225kB 8.6MB/s eta 0:00:01\r\u001b[K     |████                            | 235kB 8.6MB/s eta 0:00:01\r\u001b[K     |████▏                           | 245kB 8.6MB/s eta 0:00:01\r\u001b[K     |████▎                           | 256kB 8.6MB/s eta 0:00:01\r\u001b[K     |████▌                           | 266kB 8.6MB/s eta 0:00:01\r\u001b[K     |████▋                           | 276kB 8.6MB/s eta 0:00:01\r\u001b[K     |████▉                           | 286kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████                           | 296kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 307kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████▍                          | 317kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 327kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 337kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████▉                          | 348kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████                          | 358kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████▏                         | 368kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████▍                         | 378kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████▋                         | 389kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████▊                         | 399kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████                         | 409kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████                         | 419kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████▎                        | 430kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 440kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 450kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████▊                        | 460kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████                        | 471kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 481kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████▎                       | 491kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████▌                       | 501kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████▋                       | 512kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████▉                       | 522kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████                       | 532kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████▏                      | 542kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 552kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 563kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████▊                      | 573kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 583kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████                      | 593kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████▏                     | 604kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 614kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████▌                     | 624kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████▊                     | 634kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 645kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████                     | 655kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████▎                    | 665kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 675kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████▋                    | 686kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 696kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████                    | 706kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████                    | 716kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 727kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████▍                   | 737kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████▋                   | 747kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 757kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 768kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████▏                  | 778kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████▎                  | 788kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████▌                  | 798kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 808kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 819kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████                  | 829kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 839kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 849kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████▌                 | 860kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 870kB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████▉                 | 880kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 890kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████▏                | 901kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████▍                | 911kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 921kB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████▊                | 931kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████                | 942kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████                | 952kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 962kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 972kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 983kB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████▊               | 993kB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.0MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████               | 1.0MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▎              | 1.0MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 1.0MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▋              | 1.0MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▉              | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▏             | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▋             | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▏            | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▍            | 1.1MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▌            | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▊            | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████            | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▌           | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 1.2MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▍          | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▊          | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▌         | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 1.3MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████         | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▎        | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▌        | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▋        | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▉        | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▏       | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▍       | 1.4MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▉       | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▏      | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▍      | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▊      | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████      | 1.5MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▎     | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▋     | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▏    | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▋    | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▉    | 1.6MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▏   | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▌   | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▊   | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████   | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 1.7MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▌  | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▉  | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▎ | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▍ | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▋ | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.8MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 1.9MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▍| 1.9MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 1.9MB 8.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.9MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.9MB 8.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.9MB 8.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.7/dist-packages (from transformers) (20.9)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.41.1)\n",
            "Collecting tokenizers<0.11,>=0.10.1\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/71/23/2ddc317b2121117bf34dd00f5b0de194158f2a44ee2bf5e47c7166878a97/tokenizers-0.10.1-cp37-cp37m-manylinux2010_x86_64.whl (3.2MB)\n",
            "\u001b[K     |████████████████████████████████| 3.2MB 52.1MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: importlib-metadata; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from transformers) (3.7.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 55.2MB/s \n",
            "\u001b[?25hRequirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: typing-extensions>=3.6.4; python_version < \"3.8\" in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.7.4.3)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata; python_version < \"3.8\"->transformers) (3.4.1)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.7/dist-packages (from sacremoses->transformers) (1.0.1)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp37-none-any.whl size=893262 sha256=4b4a9b60cd678f5a77cd7bd12978bb202e72f83de52fe0b124da2686d78a82de\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.10.1 transformers-4.3.3\n",
            "Requirement already satisfied: torch in /usr/local/lib/python3.7/dist-packages (1.8.0+cu101)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.7/dist-packages (from torch) (1.19.5)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.7/dist-packages (from torch) (3.7.4.3)\n",
            "Collecting emoji\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/24/fa/b3368f41b95a286f8d300e323449ab4e86b85334c2e0b477e94422b8ed0f/emoji-1.2.0-py3-none-any.whl (131kB)\n",
            "\u001b[K     |████████████████████████████████| 133kB 8.4MB/s \n",
            "\u001b[?25hInstalling collected packages: emoji\n",
            "Successfully installed emoji-1.2.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers\n",
        "!pip install torch\n",
        "!pip install emoji"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4FaLWxNckcCp",
        "outputId": "5cbe9d85-7b73-4ed8-b88c-15bb1dd27f31"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content\n",
            "Mounted at ./MyDrive\n"
          ]
        }
      ],
      "source": [
        "import os\n",
        "print(os.getcwd())\n",
        "\n",
        "# 실행시 등장하는 URL을 클릭하여 허용해주면 인증KEY가 나타난다. 복사하여 URL아래 빈칸에 붙여넣으면 마운트에 성공하게된다.\n",
        "from google.colab import drive\n",
        "drive.mount('./MyDrive')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NTiePp9YlA8R",
        "outputId": "a4d76b5f-378a-4f6a-b85f-3467f7b5d9a0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/MyDrive/MyDrive/Capstone/code_data\n"
          ]
        }
      ],
      "source": [
        "cd MyDrive/MyDrive/Capstone/code_data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "TAXZNl6bmM7J",
        "outputId": "702274cc-c690-4697-c215-365faf436b8c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "/content/MyDrive/MyDrive/Capstone/code_data\n"
          ]
        }
      ],
      "source": [
        "!pwd"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vtIMlTiMUFzS"
      },
      "source": [
        "## TFIDF"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "adkKLRH4kI_7",
        "outputId": "b2382fb4-6ff7-4030-842a-d1e15e6bc565"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        }
      ],
      "source": [
        "import nltk\n",
        "# Uncomment to download \"stopwords\"\n",
        "nltk.download(\"stopwords\")\n",
        "from nltk.corpus import stopwords\n",
        "\n",
        "def text_preprocessing(s):\n",
        "    \"\"\"\n",
        "    - Lowercase the sentence\n",
        "    - Change \"'t\" to \"not\"\n",
        "    - Remove \"@name\"\n",
        "    - Isolate and remove punctuations except \"?\"\n",
        "    - Remove other special characters\n",
        "    - Remove stop words except \"not\" and \"can\"\n",
        "    - Remove trailing whitespace\n",
        "    \"\"\"\n",
        "    s = s.lower()\n",
        "    # Change 't to 'not'\n",
        "    s = re.sub(r\"\\'t\", \" not\", s)\n",
        "    # Remove @name\n",
        "    s = re.sub(r'(@.*?)[\\s]', ' ', s)\n",
        "    # Isolate and remove punctuations except '?'\n",
        "    s = re.sub(r'([\\'\\\"\\.\\(\\)\\!\\?\\\\\\/\\,])', r' \\1 ', s)\n",
        "    s = re.sub(r'[^\\w\\s\\?]', ' ', s)\n",
        "    # Remove some special characters\n",
        "    s = re.sub(r'([\\;\\:\\|•«\\n])', ' ', s)\n",
        "    # Remove stopwords except 'not' and 'can'\n",
        "    s = \" \".join([word for word in s.split()\n",
        "                  if word not in stopwords.words('english')\n",
        "                  or word in ['not', 'can']])\n",
        "    # Remove trailing whitespace\n",
        "    s = re.sub(r'\\s+', ' ', s).strip()\n",
        "    \n",
        "    return s"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YahIdy3XkI_9",
        "outputId": "0e75a5bd-c64c-4394-a1c0-709ae6ef0ccc"
      },
      "outputs": [
        {
          "ename": "KeyboardInterrupt",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-21-838a357a3484>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mget_ipython\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrun_cell_magic\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'time'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m''\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'from sklearn.feature_extraction.text import TfidfVectorizer\\n\\n# Preprocess text\\nX_train_preprocessed = np.array([text_preprocessing(text) for text in X])\\nX_val_preprocessed = np.array([text_preprocessing(text) for text in test_X])\\n\\n# Calculate TF-IDF\\ntf_idf = TfidfVectorizer(ngram_range=(1, 3),\\n                         binary=True,\\n                         smooth_idf=False)\\nX_train_tfidf = tf_idf.fit_transform(X_train_preprocessed)\\nX_val_tfidf = tf_idf.transform(X_val_preprocessed)'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/IPython/core/interactiveshell.py\u001b[0m in \u001b[0;36mrun_cell_magic\u001b[0;34m(self, magic_name, line, cell)\u001b[0m\n\u001b[1;32m   2115\u001b[0m             \u001b[0mmagic_arg_s\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mvar_expand\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mline\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstack_depth\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2116\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mbuiltin_trap\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 2117\u001b[0;31m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mfn\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmagic_arg_s\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mcell\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   2118\u001b[0m             \u001b[0;32mreturn\u001b[0m \u001b[0mresult\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   2119\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<decorator-gen-53>\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/IPython/core/magic.py\u001b[0m in \u001b[0;36m<lambda>\u001b[0;34m(f, *a, **k)\u001b[0m\n\u001b[1;32m    186\u001b[0m     \u001b[0;31m# but it's overkill for just that one bit of state.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    187\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mmagic_deco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 188\u001b[0;31m         \u001b[0mcall\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mlambda\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mk\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    189\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    190\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mcallable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0marg\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/IPython/core/magics/execution.py\u001b[0m in \u001b[0;36mtime\u001b[0;34m(self, line, cell, local_ns)\u001b[0m\n\u001b[1;32m   1191\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1192\u001b[0m             \u001b[0mst\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1193\u001b[0;31m             \u001b[0mexec\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcode\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mglob\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mlocal_ns\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1194\u001b[0m             \u001b[0mend\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mclock2\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1195\u001b[0m             \u001b[0mout\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;32mNone\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n",
            "\u001b[0;32m<timed exec>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n",
            "\u001b[0;32m<ipython-input-19-88ed875ee9d6>\u001b[0m in \u001b[0;36mtext_preprocessing\u001b[0;34m(s)\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mre\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msub\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34mr'([\\;\\:\\|•«\\n])'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m' '\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0ms\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0;31m# Remove stopwords except 'not' and 'can'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 27\u001b[0;31m     s = \" \".join([word for word in s.split()\n\u001b[0m\u001b[1;32m     28\u001b[0m                   \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     29\u001b[0m                   or word in ['not', 'can']])\n",
            "\u001b[0;32m<ipython-input-19-88ed875ee9d6>\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     26\u001b[0m     \u001b[0;31m# Remove stopwords except 'not' and 'can'\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     27\u001b[0m     s = \" \".join([word for word in s.split()\n\u001b[0;32m---> 28\u001b[0;31m                   \u001b[0;32mif\u001b[0m \u001b[0mword\u001b[0m \u001b[0;32mnot\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mstopwords\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'english'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     29\u001b[0m                   or word in ['not', 'can']])\n\u001b[1;32m     30\u001b[0m     \u001b[0;31m# Remove trailing whitespace\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/nltk/corpus/reader/wordlist.py\u001b[0m in \u001b[0;36mwords\u001b[0;34m(self, fileids, ignore_lines_startswith)\u001b[0m\n\u001b[1;32m     20\u001b[0m     \"\"\"\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_lines_startswith\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 22\u001b[0;31m         return [line for line in line_tokenize(self.raw(fileids))\n\u001b[0m\u001b[1;32m     23\u001b[0m                 if not line.startswith(ignore_lines_startswith)]\n\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;32m/usr/local/lib/python3.7/dist-packages/nltk/corpus/reader/wordlist.py\u001b[0m in \u001b[0;36m<listcomp>\u001b[0;34m(.0)\u001b[0m\n\u001b[1;32m     21\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mwords\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mignore_lines_startswith\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'\\n'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     22\u001b[0m         return [line for line in line_tokenize(self.raw(fileids))\n\u001b[0;32m---> 23\u001b[0;31m                 if not line.startswith(ignore_lines_startswith)]\n\u001b[0m\u001b[1;32m     24\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     25\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mraw\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mfileids\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
          ]
        }
      ],
      "source": [
        "%%time\n",
        "from sklearn.feature_extraction.text import TfidfVectorizer\n",
        "\n",
        "# Preprocess text\n",
        "X_train_preprocessed = np.array([text_preprocessing(text) for text in X])\n",
        "X_val_preprocessed = np.array([text_preprocessing(text) for text in test_X])\n",
        "\n",
        "# Calculate TF-IDF\n",
        "tf_idf = TfidfVectorizer(ngram_range=(1, 3),\n",
        "                         binary=True,\n",
        "                         smooth_idf=False)\n",
        "X_train_tfidf = tf_idf.fit_transform(X_train_preprocessed)\n",
        "X_val_tfidf = tf_idf.transform(X_val_preprocessed)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "86QdYvNrkI_9",
        "outputId": "4884b136-aa77-410d-f6e9-e0e201376b02"
      },
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "ignored",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-11-6ca3cc28359f>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mX_train_tfidf\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'X_train' is not defined"
          ]
        }
      ],
      "source": [
        "print(X_train.shape)\n",
        "print(X_train_tfidf.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CLOxl1JDkI_9"
      },
      "outputs": [],
      "source": [
        "from sklearn.model_selection import StratifiedKFold, cross_val_score\n",
        "\n",
        "def get_auc_CV(model):\n",
        "    \"\"\"\n",
        "    Return the average AUC score from cross-validation.\n",
        "    \"\"\"\n",
        "    # Set KFold to shuffle data before the split\n",
        "    kf = StratifiedKFold(5, shuffle=True, random_state=1)\n",
        "\n",
        "    # Get AUC scores\n",
        "    auc = cross_val_score(\n",
        "        model, X_train_tfidf, y_train, scoring=\"roc_auc\", cv=kf)\n",
        "\n",
        "    return auc.mean()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 311
        },
        "id": "3Bi1JS2IkI_9",
        "outputId": "6c986067-c6ca-4f99-9732-d2302aed588e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Best alpha:  1.0\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAY4AAAEWCAYAAABxMXBSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3yV5f3/8dcng7BBCDssAdk7ggIKglsUoaigIg609ovVam2rP+vXfm0drbau2roF3CgqaF3IcoBKmIIMw04AE6YMWcnn98e5sTGyAufkTnLez8fjPHLOdY987vPQvLnucV3m7oiIiByphLALEBGR0kXBISIiRaLgEBGRIlFwiIhIkSg4RESkSBQcIiJSJAoOkVLOzK40s8+iva7IwSg4pMwzs6lmttnMUg7QPqJQWx8zyyrw2czsRjNbYGY7zCzLzF43s/bFVX+BWv5kZm5m3Yv7d4sUpOCQMs3MmgCnAA5ccBS7eAS4CbgRqAGcALwNnBedCo+MmRlwBbAp+CkSGgWHlHVXAF8Ao4DhRdnQzFoAI4Gh7j7Z3Xe7+053f8nd7z/A+peYWUahtpvNbELw/lwz+8bMtplZtpndWoRyTgHqEQmwIWZW7hB1e9BLWm5mG8zsATNLKLTOg0EvbIWZnVOg/SozWxTUuNzMflmEGiVOKDikrLsCeCl4nWVmdYqwbT8gy92/OsL13wFaBoGz36XAy8H7Z4FfunsVoB0wuQi1DA/2Pzb4fP5h1h8IpANdgAHA1QWWdQeWAKnA34Bngx4NQA7QH6gKXAU8ZGZdilCnxAEFh5RZZtYLaAyMdfdZwDIif8iPVE1g3ZGu7O47gfHA0OD3twBaAROCVfYCbcysqrtvdvfZR7JfM6sIXAS87O57gTc4/Omqv7r7JndfDTy8v6bAKnd/2t3zgNFEejJ1gmP4j7sv84hpwEdEejsiP1JwSFk2HPjI3TcEn1/mp6er9gHJhbZJJvIHHmAjkT+qRfEy//0jfSnwdhAoAL8AzgVWmdk0Mzv5CPc5MKj1veDzS8A5ZlbrENusKfB+FVC/wOf1+98UqK0ygJmdY2ZfmNkmM9sS1Jt6hHVKnFBwSJlkZhWAi4HeZrbezNYDNwMdzaxjsNpqoEmhTZsS+UMLMAlIM7P0IvzqiUAtM+tEJED2n6bC3We6+wCgNpEL7GMPvIufGU7kD/vq4DheJxJwh+o9NSzwvhGw9nC/JLjrbBzwIFDH3asTCSs75IYSdxQcUlZdCOQBbYBOwas18Cn/Pc3zGnCVmXULbrs9gUi4vArg7t8C/wJeCW7TLWdm5c1siJnddqBfGpxKeh14gMhdWBMBgm0vM7NqwTrfA/mHOwgza0DkWkv/AsfREfgrhz5d9TszO87MGhK5K+y1w/0uoByQAuQC+4KL5mcewXYSZxQcUlYNB55399Xuvn7/C/gncJmZJbn7h8BtwPPAViL/uh4NPFVgPzcG2zwObCFynWQgkQvVB/MycDrwurvvK9A+DFhpZt8D1wOXAZhZIzPbbmaNDrCvYcBcd/+o0HE8CnQws3YHqWE8MAuYC/yHyIX5Q3L3bcHxjgU2E+nRTDjkRhKXTBM5iZQtZuZAC3fPDLsWKZvU4xARkSJRcIiISJHoVJWIiBSJehwiIlIkSWEXUBxSU1O9SZMmYZchIlKqzJo1a4O7/+xB07gIjiZNmpCRkXH4FUVE5EdmtupA7TpVJSIiRaLgEBGRIolpcJjZ2Wa2xMwyDzREg5k1NrNJZjY/mI0tLWjvZGYzzGxhsOySAts0NbMvg32+dqh5CUREJPpiFhxmlkhkmIZziIwXNNTM2hRa7UFgjLt3AO4G7gvadwJXuHtb4GzgYTOrHiz7K/CQuzcnMizCNbE6BhER+blY9ji6AZnuvtzd9xAZOG5AoXXa8N/JbKbsX+7uS4MB5nD3tUQml6kVTDbTl8h8BBAZV+jCGB6DiIgUEsvgaMBP5wTICtoKmgcMCt4PBKqYWc2CK5hZNyKjdi4jMrHOlgIDxx1on/u3u87MMswsIzc395gORERE/ivsi+O3EpkvYQ7QG8gmMhQ2AGZWD3gBuMrdDzsEdUHu/pS7p7t7eq1ah5rvRkREiiKWz3Fk89PJZNKCth8Fp6EGAZhZZeAX7r4l+FyVyHDQd7j7F8EmG4HqwZDY+w60z2j6+JvvyN2+m6HdDjTatYhIfIplj2Mm0CK4C6ocMIRCY/ubWaqZ7a/hduC5oL0c8BaRC+f7r2fgkYG1pgCDg6bhROYdiIlXZ67hTxMWsix3e6x+hYhIqROz4Ah6BDcAHwKLgLHuvtDM7jazC4LV+gBLzGwpUAe4J2i/GDgVuNLM5gavTsGyPwC3mFkmkWseh52g5mjdO7AdKUkJ/P6N+eTlazBIERGIk9Fx09PT/WiHHHlzdha3jJ3Hnf3bcE2vplGuTESk5DKzWe6eXrg97IvjJd7Azg3o16o2D3y4mBUbdoRdjohI6BQch2Fm3DuoPeUSE/j9G/PI1ykrEYlzCo4jUKdqee7s34aZKzfz0lerwy5HRCRUCo4jNLhrGj2b1+Rv7y8m5/tdYZcjIhIaBccRMjP+cmF7duflc/e734RdjohIaBQcRdA0tRI3nNacd+evY8qSnLDLEREJhYKjiH7Z+3ia1arEnW8v4Ic9eYffQESkjFFwFFFKUiL3DmxP1uYf+MfEJWGXIyJS7BQcR6H78TW5tHsjnvlsBTOWbQy7HBGRYqXgOEp/PK81TWtW4paxc9m6c2/Y5YiIFBsFx1GqWC6Jh4d0Infbbv44fgHxMHSLiAgoOI5Jh7Tq/Ob0Frwzby1vz43Z6O4iIiWKguMY/apPc05schx3vr2Qpd9tC7scEZGYU3Aco8QE45EhnalQLpGrnp9J7rbdYZckIhJTCo4oqF+9As8OT2fjjt2MGJOh5ztEpExTcERJh7TqPDKkM/OztnDza3M1iq6IlFkKjig6q21d7ji3NR8sXM8jk74NuxwRkZhQcETZNb2aMrBzAx6b/C2zVm0KuxwRkahTcESZmfF/A9pSv3oFfvPaXLbt0sOBIlK2KDhioGr5ZB6+pBPZm3/grgkLwy5HRCSqFBwxkt6kBjec1pw3Z2fz7vy1YZcjIhI1Co4Y+nW/FnRqWJ3bx31NZo4eDhSRskHBEUPJiQk8flkXUpITGDE6gy0794RdkojIMVNwxFiD6hV44vKuZG/5gZEvz2ZvXn7YJYmIHBMFRzFIb1KDewe25/PMjfxF85WLSCmXFHYB8eKi9IYsWb+NZz5bQfPalRl2cpOwSxIROSoKjmJ0+7mtWbFhB3dNWEiD4yrQt1WdsEsSESkynaoqRokJxqNDO9O6XlVueHkOC7K3hl2SiEiRKTiKWaWUJJ678kSqV0jm6lEzWbvlh7BLEhEpEgVHCOpULc9zV53Izj15XD1qpoYlEZFSRcERklZ1q/Lvy7vwbc52Rr48R7fpikipoeAI0SktanHPhe34ZGku/zt+Ie6aw0NESr6YBoeZnW1mS8ws08xuO8DyxmY2yczmm9lUM0srsOwDM9tiZu8W2maUma0ws7nBq1MsjyHWhnRrxK/6NOOVr1bz5CfLwy5HROSwYhYcZpYIPA6cA7QBhppZm0KrPQiMcfcOwN3AfQWWPQAMO8juf+funYLX3CiXXux+d2ZL+neox/3vL+a9r9eFXY6IyCHFssfRDch09+Xuvgd4FRhQaJ02wOTg/ZSCy919EhAXIwMmJBgPXtSRLo2qc/Nrc5m7ZkvYJYmIHFQsg6MBsKbA56ygraB5wKDg/UCgipnVPIJ93xOc3nrIzFIOtIKZXWdmGWaWkZubW9Tai1355ESeviKd2lVTGDE6g6zNO8MuSUTkgMK+OH4r0NvM5gC9gWwg7zDb3A60Ak4EagB/ONBK7v6Uu6e7e3qtWrWiWHLs1KycwvNXnsjufXlcMypDt+mKSIkUy+DIBhoW+JwWtP3I3de6+yB37wzcEbQd8jyNu6/ziN3A80ROiZUZzWtX4YnLu7IsdzvXjsnghz2Hy1ERkeIVy+CYCbQws6ZmVg4YAkwouIKZpZrZ/hpuB5473E7NrF7w04ALgQVRrboE6Nk8lb9f3JGvVmxixJiZ7Nqr8BCRkiNmweHu+4AbgA+BRcBYd19oZneb2QXBan2AJWa2FKgD3LN/ezP7FHgd6GdmWWZ2VrDoJTP7GvgaSAX+EqtjCNOATg14YHBHpi/byHUvzFJ4iEiJYfHw0Fl6erpnZGSEXcZRGTtzDb8fN5++rWrz5LCuJCeGfVlKROKFmc1y9/TC7forVMJdfGJD7hnYjsmLc7jz7QV6ulxEQqf5OEqBy7o3Zv3WXTw2OZOGNSoy8rTmYZckInFMwVFK3HLGCazZtJMHPlxC2nEVGNCp8CMxIiLFQ8FRSpgZfx3cgXVbd/G71+dTp2p5Tjr+SJ6VFBGJLl3jKEVSkhJ5alg6jWpW5NoxGSxZHxcjsohICaPgKGWqVUxm9NXdqFgukeHPfaUZBEWk2Ck4SqEG1Ssw6qpu7Ni9jyuf/4qtOzU0iYgUHwVHKdW6XlWeHNaVFRt2MGLMTHbu2Rd2SSISJxQcpViP5qk8dEknZq3azLVjMvR0uYgUCwVHKde/Q30evCgyNMn1L85i9z6Fh4jEloKjDBjUJY17B7Zn6pJcbnh5Dnvz8sMuSUTKMAVHGTG0WyP+74K2TPzmO258ReEhIrGj4ChDhvdowp392/D+gvX85tW57FN4iEgM6MnxMuaaXk3Jz3fueW8RCQnGQxd3JEkj6opIFCk4yqBrTz2efHfue38xAP+4uKOGYxeRqFFwlFG/7N0MB+5/fzF79uXx2NAulEtSeIjIsdNfkjLs+t7NuOv8Nny48Duuf1GzCIpIdCg4yrirejbl3oHtmbIkhxGjM/SEuYgcMwVHHLi0eyMeHNyR6cs2MOzZr9j6g8a2EpGjp+CIE7/omsbjl3ZhftYWhj71BRu37w67JBEppRQcceSc9vV4+op0lm/YzsVPzmDdVg3JLiJFp+CIM31a1mbM1d357vvd/OJf08nM0WRQIlI0Co441K1pDV697iT25DmDn5jBnNWbwy5JREoRBUecategGuN+dTLVKiRz6dNfMmVJTtgliUgpoeCIY41rVuKN63twfK1KjBidwdiZa8IuSURKAQVHnKtVJYXXfnkyPZrV5Pfj5vPwx0tx97DLEpESTMEhVE5J4rkrT2Rw1zQe/vhb/jBuvoZlF5GD0lhVAkByYgIPDO5A/eoVeHTSt6zbuovHL+tC1fLJYZcmIiWMehzyIzPjljNO4G+/6MCMZRu56N8zyN6iZz1E5KcUHPIzF5/YkFFXdWPtlh+48PHPmZ+1JeySRKQEUXDIAfVqkcq4/+lBucQELnpiBuPnZoddkoiUEDENDjM728yWmFmmmd12gOWNzWySmc03s6lmllZg2QdmtsXM3i20TVMz+zLY52tmVi6WxxDPTqhThfE39KRjWnVuenUu97+/mLx83XElEu9iFhxmlgg8DpwDtAGGmlmbQqs9CIxx9w7A3cB9BZY9AAw7wK7/Cjzk7s2BzcA10a5d/iu1cgovjujO5Sc14olpy7hm9Ey+36XRdUXiWSx7HN2ATHdf7u57gFeBAYXWaQNMDt5PKbjc3ScBPxlIycwM6Au8ETSNBi6MfulSULmkBP5yYXvuGdiOz77dwOB/T2fNpp1hlyUiIYllcDQACj6KnBW0FTQPGBS8HwhUMbOah9hnTWCLu++fjehA+wTAzK4zswwzy8jNzS1y8fJzl3VvzJhrurF+6y4G/utzjXElEqfCvjh+K9DbzOYAvYFsICrzm7r7U+6e7u7ptWrVisYuBejRLJU3/6cnlVKSGPLUF7w7f23YJYlIMYtlcGQDDQt8TgvafuTua919kLt3Bu4I2g517+dGoLqZ7X9w8Wf7lNhrXrsyb/1PTzqkVeOGl+fwz8nfapgSkTgSy+CYCbQI7oIqBwwBJhRcwcxSzWx/DbcDzx1qhx756zQFGBw0DQfGR7VqOSI1KpXjxRHdGdi5AQ9+tJTfvj6P3fui0lkUkRIuZsERXIe4AfgQWASMdfeFZna3mV0QrNYHWGJmS4E6wD37tzezT4HXgX5mlmVmZwWL/gDcYmaZRK55PBurY5BDS0lK5B8Xd+SWM07gzdnZXP7Ml+Rs2xV2WSISYxYPpxjS09M9IyMj7DLKtHfmreV3b8yjSvlkHhvamZOOP9Q9DiJSGpjZLHdPL9we9sVxKSPO71if8SN7USUlicue+ZInpi3TdQ+RMkrBIVHTsm7kSfOz2tbh/vcXc8VzX7FWgySKlDkKDomqKuWTefzSLvz5wnbMWrWZsx76hNcz1qj3IVKGKDgk6syMYSc15oObTqV1/ar87o35/PKFWWzTUCUiZYKCQ2KmUc2KvHrtSfzxvNZMWpzDoH9NZ/VGDVUiUtopOCSmEhKMEaccz5iru5GzbTcDHv+MGcs2hl2WiBwDBYcUi57NUxk/sic1KpVj2LNf8sKMlbruIVJKHTQ4zOwsMxt8gPbBZnZGbMuSsqhJaiXeGtmTU0+oxZ3jF/KHcfPZtVdPm4uUNofqcfwvMO0A7VOJzJ0hUmRVyyfzzBXp3Ni3OWMzsrjkyRms26pbdkVKk0MFR4q7/2w8cnffAFSKXUlS1iUkGLec2ZInLu9KZs52znv0M6YuyQm7LBE5QocKjqoFRqH9kZklAxViV5LEi7Pb1WX8Db2oXSWFK5+fyf3vL2ZvXn7YZYnIYRwqON4EnjazH3sXZlYZeCJYJnLMmteuzNsjezK0W2Rq2iFPfUG2njYXKdEOFRx/BL4DVpnZLDObDawAcoNlIlFRPjmR+wa159GhnVmyfhvnPPwJHyxYH3ZZInIQhx0d18wqAM2Dj5nuXur+OajRcUuPVRt38OtX5jA/ayvDTmrMHee1pnxyYthlicSlg42O+7NrGAU2GFSoyYnMvjfX3bdFu0ARgMY1K/HG9T144MPFPP3pCmav3swTl3elYY2KYZcmIoGD9jjM7PkDNNcAOgDXuPvkWBYWTepxlE6TFn3Hza/Nxcx4eEgnTmtZO+ySROLKwXocRZ7IycwaE5nNr3u0ios1BUfptWrjDq5/cTaL13/Pr09rzg19W1AuSQMeiBSHqE3k5O6rgOSoVCVyGI1rVuLNX/VgYOcGPDo5k/Me/ZSvVmwKuyyRuFbk4DCzVsDuGNQickAVyiXyj4s78ezwdHbuyePiJ2fwu9fnsWnHnrBLE4lLh7o4/g6RC+IF1QDqAZfHsiiRA+nXug4nN6vJY5MzefqT5UxanMOd/VtzYacGmFnY5YnEjUNdHO9dqMmBTUTC4xJ3Hxnj2qJG1zjKniXrt3Hbm/OZs3oLp7RI5S8XtqNxTY2EIxJNRb7G4e7T9r+A74HzgXeB/wMWxaxSkSPQsm4Vxl3fgz8PaMuc1Vs46+FPePazFeTla6h2kVg71LDqJ5jZXWa2GHgMWE2kh3Kau/+z2CoUOYiEBGPYyU2YeMup9GiWyp/f/YaLn5xBZs72sEsTKdMOdXF8MdAX6O/uvdz9MUCTJ0iJU69aBZ4dns5Dl3QkM2c75z76KU9OW6beh0iMHCo4BgHrgClm9rSZ9QN0BVJKJDNjYOc0Jt5yKn1OqMV97y/moiemsyxXvQ+RaDvUNY633X0I0AqYAvwGqG1m/zazM4urQJGiqF2lPE8O68ojQzqxLHcH5z7yKaM+X6FpakWi6LDPcbj7Dnd/2d3PB9KAOcAfYl6ZyFEyMwZ0asDEm0+lZ/NU/vTON1w7Zhab9dyHSFQU6QFAd9/s7k+5e79YFSQSLbWrlufZ4en8b/82TFuaw7l66lwkKjToj5RpZsbVvZry5q96Ui4pgUuemsFd4xfw/a69YZcmUmopOCQutE+rxn9uPIXhJzfhhS9W0e/v05gwb62ufYgcBQWHxI3KKUn86YK2jB/Zi3rVynPjK3O4atRM1m/dFXZpIqWKgkPiTvu0arz1Pz256/w2fLl8E2c8NI3XM9ao9yFyhGIaHGZ2tpktMbNMM7vtAMsbm9kkM5tvZlPNLK3AsuFm9m3wGl6gfWqwz7nBS7P7SJElJhhX9WzKB785hdZ1q/K7N+Zz1aiZrN64M+zSREq8Ik/kdMQ7NksElgJnAFnATGCou39TYJ3XgXfdfbSZ9QWucvdhZlYDyADSiQyuOAvo6u6bzWwqcKu7H/GohRrkUA4lP98ZNX0lf/9oCfvynZGnNee6U4/XXOcS96I2kVMRdAMy3X25u+8BXgUGFFqnDbB/CtopBZafBUx0903uvhmYCJwdw1oljiUkRO68mvTbPpzepg7/mLiUsx/+hCmLc8IuTaREimVwNADWFPicFbQVNI/I0CYAA4EqZlbzCLZ9PjhNdacdZCIGM7vOzDLMLCM3N/dYjkPiRN1q5Xn80i68eE13EhKMq0bN5JpRM1m5YUfYpYmUKGFfHL8V6G1mc4DeQDaHH0jxMndvD5wSvIYdaKXgQcV0d0+vVatWNGuWMq5Xi1Q+uOlU7ji3NV+u2MSZD33Cfe8v0rMfIoFYBkc20LDA57Sg7UfuvtbdB7l7Z+COoG3LobZ19/0/twEvEzklJhJV5ZISuPbU45l8a28u6FSfpz5ZTp8HpjJ6+kr25uWHXZ5IqGIZHDOBFmbW1MzKAUOACQVXMLNUM9tfw+3Ac8H7D4Ezzew4MzsOOBP40MySzCw12DYZ6A8siOExSJyrXaU8D17UkXdu6EWrulW4a8JCznroEz7P3BB2aSKhiVlwuPs+4AYiIbAIGOvuC83sbjO7IFitD7DEzJYCdYB7gm03AX8mEj4zgbuDthQiATIfmEukF/J0rI5BZL92Darx0ojuPHdlOvnuXPbMl9wydi6bNHCixKGY3Y5bkuh2XImmXXvz+OfkTJ6Ytowq5ZO46/y2DOhUn4PcpyFSaoVxO65ImVQ+OZFbz2rJf248hSaplfjNa3P51Yuz2bh9d9iliRQLBYfIUWpZtwpvXN+D285pxeTFOZz50Cd8sGBd2GWJxJyCQ+QYJCYY1/duxju/7kXdauW5/sXZXKOhS6SMU3CIREHLulV4e2RP7ji3NV8s38jpD03j4Y+Xsmvv4R5LEil9FBwiUZKcGHn2Y9Jv+3BW27o8/PG39H5gCi99uUrPfkiZouAQibK61crz2NDOvHbdSaQdV5E73lrA6f+YxjuaOErKCAWHSIx0P74mb1x/Ms8OT6dCciK/fmUOlz79Jd9+ty3s0kSOiYJDJIbMjH6t6/CfG0/hzxe2Y+HarZzzyKfc9/4iduzeF3Z5IkdFwSFSDBITjGEnNWbKrX0Y2LkBT05bTt+/T2X83GydvpJSR8EhUoxqVk7hgYs6Mu5XPahdpTw3vTqXi5+cwddZW8MuTeSIKThEQtC18XG8PbIn9w9qz7LcHZz/z88YMXqmAkRKBY1VJRKy73ftZfTnK3nmsxVs/WEvfVvV5qZ+LejYsHrYpUmcO9hYVQoOkRJi2669jJ6+kqc/jQRIn5a1uLFfC7o0Oi7s0iROKTgUHFJKbNu1lxe+WMXTnyxn8869nHpCLW4+vQWdFSBSzBQcCg4pZXbs3seLX6ziyU+Ws2nHHvq0rMXNp5+gU1hSbBQcCg4ppXbs3seYGat48pNlbNkZOYV1Uz/1QCT2FBwKDinltu/ex5gZK388hdX7hFr8/uyWtK1fLezSpIxScCg4pIzYvnsfL8xYxVOfLGPLD3u5JL0hvz2zJbWqpIRdmpQxCg4Fh5QxW3/Yy2OTvmXU9JWUT07k+t7Hc0WPJlQtnxx2aVJGKDgUHFJGLc/dzr3vLeLjRTlUSUni8pMbc3XPpuqByDFTcCg4pIxbkL2Vf09bxntfr6NcYgLDTmrM9X2akVpZASJHR8Gh4JA4sWLDDv45OZO35mRRPjmRq3o2YUSv4zmuUrmwS5NSRsGh4JA4syx3Ow9//C3vzFtLSlICF3ZqwPAeTWhTv2rYpUkpoeBQcEicWrJ+G6Omr+CtOdns2ptPtyY1GNm3Oae2SMXMwi5PSjAFh4JD4tzWnXsZm7GG5z9fwdqtu+iYVo0b+7Wgb6vaChA5IAWHgkMEgD378hk3O4t/Tc1kzaYfaF67MsNPbszALmlUTkkKuzwpQRQcCg6Rn9ibl88789YyavpK5mdtpXJKEpec2JBf6U4sCSg4FBwiBzVn9WZGT1/JhHlrqZCcyIhTjmfEKU2poocJ45qCQ8EhcliZOdv5x8QlvPf1empUKseVPZpwWfdG1FQPJC4pOBQcIkds3potPPzxUqYsySUlKYFBXdK4pldTmteuHHZpUowUHAoOkSLLzNnGs5+t5M3ZWezel8/prWvzy97NSG98nO7EigMKDgWHyFHbuH03Y2asYsyMlWzeuZfOjaozsk9z+rXWrbxl2cGCIyHGv/RsM1tiZplmdtsBljc2s0lmNt/MpppZWoFlw83s2+A1vEB7VzP7Otjno6b/akVirmblFG4+4wSm39aPuwe0JXfbbkaMyeDcRz/jva/XkZ9f9v8BKv8Vsx6HmSUCS4EzgCxgJjDU3b8psM7rwLvuPtrM+gJXufswM6sBZADpgAOzgK7uvtnMvgJuBL4E3gMedff3D1WLehwi0bU3L5/xc9fyrymZLN+wg+a1KzPytGac36E+SYkx/feoFKMwehzdgEx3X+7ue4BXgQGF1mkDTA7eTymw/CxgortvcvfNwETgbDOrB1R19y88knhjgAtjeAwicgDJiQkM7prGxFt68+jQziSacfNr8+j792m88tVq9uzLD7tEiaFYBkcDYE2Bz1lBW0HzgEHB+4FAFTOreYhtGwTvD7VPAMzsOjPLMLOM3Nzcoz4IETm4xATjgo71ef+mU3hqWFeOq5jM7W9+Td+/T+XVr1azN08BUhaF3ae8FehtZnOA3kA2kBeNHbv7U+6e7u7ptWrVisYuReQgEhKMM9vW5e2RPXn+qhOpWakctwUB8syny9m0Y0/YJUoUxXJgmmygYYHPaUHbj9x9LUGPw8wqA79w9y1mlg30KbTt1GD7tELtP9mniITHzDitZW36nFCLKUtyeGxyJn/5zyL+9sESzmxbh4AUbUQAAAyESURBVEu7NeLkZjV1J1YpF8vgmAm0MLOmRP64DwEuLbiCmaUCm9w9H7gdeC5Y9CFwr5kdF3w+E7jd3TeZ2fdmdhKRi+NXAI/F8BhE5CiYGX1b1aFvqzosXv89r361hrfmZPPu/HU0q1WJYSc1ZlDXNM2PXkrF9DkOMzsXeBhIBJ5z93vM7G4gw90nmNlg4D4id059Aox0993BtlcD/y/Y1T3u/nzQng6MAioA7wO/9sMchO6qEgnfrr15/Gf+Ol74YhVz12yhYrlEhp3cmF+e2owamp2wRNIDgAoOkRLj66ytPPPZ8h8HVRzeowkjejXVmFgljIJDwSFS4mTmbOORSZm8O38tyYkJ9G9fj2EnN6ZTw+q6DlICKDgUHCIlVmbONkZPX8Wbs7PYsSeP9g2qcWWPJvTvWI+UpMSwy4tbCg4Fh0iJt333Pt6ancXoGavIzNlOauVyDO3WiMtPakydquXDLi/uKDgUHCKlhrvzeeZGRk1fwaTFOSSacXa7ulzZowldNTJvsTlYcGiCYREpccyMXi1S6dUildUbdzJmxkpey1jDu/PX0bpeVX7RpQEXdKxPbfVCQqEeh4iUCjv37OOtOdmMnbmGeVlbSTDo2TyVod0acUabOiRrcMWo06kqBYdImbEsdztvz8nmzdnZZG/5gdpVUhjarRFDuzWibjX1QqJFwaHgEClz8vKdqUtyeOGLVUxbmkuCGae3rs1l3RvTq3kqCQm6FnIsdI1DRMqcxASjX+s69Gtdh1Ubd/DKV2sYm7GGDxd+R6MaFRncNY1BXRqQdlzFsEstU9TjEJEyZfe+PD5c+B2vfLmaGcs3AtCjWU0GdUnjrLZ1qKLxsY6YTlUpOETizppNO3lzdjbjZmexetNOUpISOKNNHS7oWJ9TWtSiQjk9XHgoCg4Fh0jccndmr97M23PW8u78tWzeuZeUpAR6Nk+lb6va9O9Qj+oVNdBiYQoOBYeIEJkv/YvlG5m8OIdJi3JYvWknFZITuSg9jat7NqVJaqWwSywxFBwKDhEpxN35Zt33jPp8JePnrmVvfj59W9bmovSG9G1Vm3JJ8f1siIJDwSEih5CzbRcvzFjFazPXkLNtNzUqlWNAp/oM7daIE+pUCbu8UCg4FBwicgT25eXzaeYG3sjIYuI337EnL5/0xsdxafdGnNu+HuWT4+eCuoJDwSEiRbRpxx7Gzcri5a9Ws2LDDiqVS+SMNnU4r0N9TmmRWuZDRMGh4BCRo+TuzFi+kQlz1/LBwvVs2bmXKilJnNm2LgM61adHs5oklcGxshQcCg4RiYK9eflMX7aRd+at5cMF69m2ex+plcvRv0N9LkpPo239amGXGDUKDgWHiETZrr15TF2Sy4R52Xz8TQ578vJpU68qg7umMaBT/VI/h7qCQ8EhIjG0Zece3pm3ltdnZTE/aytJCcZprWozuGsap7Usnbf2KjgUHCJSTJas38a42Vm8NSeb3G27qVYhmXPa1eWCTvXp3rQmiaVk1F4Fh4JDRIrZ/lt7J8xdy0cL17NjTx4NqlfgipMbc8mJDUv8MCcKDgWHiITohz15fLzoO176chVfLN9E+eQEBnZOY2DnBqQ3Pq5Ezh2i4FBwiEgJsWjd94yevpK35mSze18+taukcE67upzXoX6JChEFh4JDREqYHbv3MWlxDu/NX8eUJTns3pdPnaopnNu+Hv071KdLo+qYhRciCg4Fh4iUYPtD5N15a5m6NJc9+/JpXLMigzpHZjFsWKP4ZzFUcCg4RKSU2LZrLx8sWM9bc7KZsXwj7tClUXXO61Cfc9vXpV61CsVSh4JDwSEipVD2lh94e042/5m/jm/WfQ9Ax7RqnHR8Tbo1rUF64xpUqxib6XAVHAoOESnlludu572v1zFtaS7z1mxlT14+ZnBi4xqc074uZ7eLbm9EwaHgEJEyZNfePOau2cL0zA18uPA7lny3DYCODatzVts6nNmmLs1rVz6m3xFKcJjZ2cAjQCLwjLvfX2h5I2A0UD1Y5zZ3f8/MygFPAulAPnCTu08NtpkK1AN+CHZzprvnHKoOBYeIlHXLcrfzwYL1fLRwPfOytgJwfK1KPHF516OeiOpgwZF0bKUe8hcmAo8DZwBZwEwzm+Du3xRY7Y/AWHf/t5m1Ad4DmgDXArh7ezOrDbxvZie6e36w3WXuriQQEQk0q1WZkac1Z+RpzVm39QcmfvMdkxbl0KB69C+kx3LUrW5Aprsvd/c9wKvAgELrOFA1eF8NWBu8bwNMBgh6E1uI9D5EROQw6lWrwBUnN2H01d2olBL9/kEsg6MBsKbA56ygraA/AZebWRaR3savg/Z5wAVmlmRmTYGuQMMC2z1vZnPN7E47yNMxZnadmWWYWUZubm4UDkdERCC2wXEkhgKj3D0NOBd4wcwSgOeIBE0G8DAwHcgLtrnM3dsDpwSvYQfasbs/5e7p7p5eq1atGB+GiEj8iGVwZPPTXkJa0FbQNcBYAHefAZQHUt19n7vf7O6d3H0AkYvnS4P1soOf24CXiZwSExGRYhLL4JgJtDCzpsFdUkOACYXWWQ30AzCz1kSCI9fMKppZpaD9DGCfu38TnLpKDdqTgf7Aghgeg4iIFBKzu6rcfZ+Z3QB8SORW2+fcfaGZ3Q1kuPsE4LfA02Z2M5EL5Ve6uwd3Un1oZvlEein7T0elBO3JwT4/Bp6O1TGIiMjP6QFAERE5oIM9xxH2xXERESllFBwiIlIkcXGqysxygVVh13GMUoENYRdRwug7+Sl9Hz+l7+PnivqdNHb3nz3PEBfBURaYWcaBzjXGM30nP6Xv46f0ffxctL4TnaoSEZEiUXCIiEiRKDhKj6fCLqAE0nfyU/o+fkrfx89F5TvRNQ4RESkS9ThERKRIFBwiIlIkCo4SzswamtkUM/vGzBaa2U1h11QSmFmimc0xs3fDrqUkMLPqZvaGmS02s0VmdnLYNYXJzG4O/n9ZYGavmFn5sGsqbmb2nJnlmNmCAm01zGyimX0b/DzuaPat4Cj59gG/dfc2wEnAyGCa3Xh3E7Ao7CJKkEeAD9y9FdCROP5uzKwBcCOQ7u7tiAyIOiTcqkIxCji7UNttwCR3bwFMCj4XmYKjhHP3de4+O3i/jcgfhMIzKcYVM0sDzgOeCbuWksDMqgGnAs8CuPsed98SblWhSwIqmFkSUJH/TksdN9z9E2BToeYBwOjg/WjgwqPZt4KjFDGzJkBn4MtwKwndw8DvgfywCykhmgK5RKZUnmNmz+yfzyYeBZO9PUhkvp91wFZ3/yjcqkqMOu6+Lni/HqhzNDtRcJQSZlYZGAf8xt2/D7uesJhZfyDH3WeFXUsJkgR0Af7t7p2BHRzlKYiyIDhvP4BIoNYHKpnZ5eFWVfJ45FmMo3oeQ8FRCgQTV40DXnL3N8OuJ2Q9gQvMbCXwKtDXzF4Mt6TQZQFZ7r6/J/oGkSCJV6cDK9w91933Am8CPUKuqaT4zszqAQQ/c45mJwqOEs7MjMi560Xu/o+w6wmbu9/u7mnu3oTIBc/J7h7X/5p09/XAGjNrGTT1A74JsaSwrQZOCqagNiLfR9zeLFDIBGB48H44MP5odqLgKPl6Epk6t6+ZzQ1e54ZdlJQ4vwZeMrP5QCfg3pDrCU3Q83oDmA18TeTvXNwNP2JmrwAzgJZmlmVm1wD3A2eY2bdEemb3H9W+NeSIiIgUhXocIiJSJAoOEREpEgWHiIgUiYJDRESKRMEhIiJFouAQiSIzu9DM3MxaBZ+bFByd9CDbHHYdkZJEwSESXUOBz4KfImWSgkMkSoLxxHoB13CAYbzN7EozG29mU4P5EO4qsDjRzJ4O5pD4yMwqBNtca2YzzWyemY0zs4rFczQiB6fgEImeAUTmxFgKbDSzrgdYpxvwC6ADcJGZpQftLYDH3b0tsCVYB+BNdz/R3ffPsXFNTI9A5AgoOESiZyiRgRcJfh7odNVEd9/o7j8QGXyvV9C+wt3nBu9nAU2C9+3M7FMz+xq4DGgbk8pFiiAp7AJEygIzqwH0BdqbmROZdc6BxwutWniMn/2fdxdoywMqBO9HARe6+zwzuxLoE72qRY6Oehwi0TEYeMHdG7t7E3dvCKwAGhZa74xg3ucKRGZf+/ww+60CrAuG1r8s6lWLHAUFh0h0DAXeKtQ2Dri9UNtXQft8YJy7Zxxmv3cSmfHxc2BxFOoUOWYaHVekmASnmtLd/YawaxE5FupxiIhIkajHISIiRaIeh4iIFImCQ0REikTBISIiRaLgEBGRIlFwiIhIkfx/dgWqYKjCsPoAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "from sklearn.naive_bayes import MultinomialNB\n",
        "\n",
        "res = pd.Series([get_auc_CV(MultinomialNB(i))\n",
        "                 for i in np.arange(1, 10, 0.1)],\n",
        "                index=np.arange(1, 10, 0.1))\n",
        "\n",
        "best_alpha = np.round(res.idxmax(), 2)\n",
        "print('Best alpha: ', best_alpha)\n",
        "\n",
        "plt.plot(res)\n",
        "plt.title('AUC vs. Alpha')\n",
        "plt.xlabel('Alpha')\n",
        "plt.ylabel('AUC')\n",
        "plt.show()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 327
        },
        "id": "PZi8GU6dkI_-",
        "outputId": "eae6e8c0-0ffe-49aa-922c-ab0ab749b5e4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AUC: 0.9015\n",
            "Accuracy: 82.10%\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3de7xVc/7H8ddHuqDElDGmC41CF0kiueUWScREco9oxmXcDcaMMcaMMQzDjEu5DGOoIaNySX5UEkI36SJSdCGSolC6fH5/fNd2dsc5++xzWXvtvc/7+XjsR3vtvfZan706e3/2d32/6/M1d0dERKQ8myUdgIiI5DclChERyUiJQkREMlKiEBGRjJQoREQkIyUKERHJSIlCKsXMZpnZwUnHkS/M7Ddmdn9C+37IzG5MYt81zcxONbMXqvha/U3GTImigJnZh2b2rZmtNrOl0RdHwzj36e7t3X18nPtIMbP6ZnaTmS2M3uf7ZnalmVku9l9GPAeb2eL0x9z9z+5+Tkz7MzO7yMxmmtnXZrbYzJ4ws93j2F9Vmdn1Zvaf6mzD3R919yOy2NcPkmMu/yZrKyWKwneMuzcEOgF7AtckHE+lmdnm5Tz1BHAY0AtoBJwODALuiCEGM7N8+zzcAVwMXAT8CNgFGAEcXdM7yvB/ELsk9y1ZcnfdCvQGfAgcnrb8V+DZtOV9gdeAlcDbwMFpz/0I+BfwMbACGJH2XG9gevS614COpfcJ/BT4FvhR2nN7Ap8DdaPls4E50fbHADumrevABcD7wIIy3tthwBqgRanHuwIbgNbR8njgJuBN4CtgZKmYMh2D8cCfgFej99IaOCuKeRUwH/hFtO5W0TobgdXR7afA9cB/onV2it7XmcDC6Fhcm7a/LYCHo+MxB/g1sLic/9s20fvcJ8P//0PAXcCzUbxvADunPX8HsCg6LlOAA9Oeux4YDvwnev4cYB/g9ehYfQL8E6iX9pr2wP8BXwCfAr8BegLfAeuiY/J2tG5j4IFoO0uAG4E60XMDomN+O7A8em4AMDF63qLnPotiewfoQPiRsC7a32rg6dKfA6BOFNcH0TGZQqm/Id2q8F2TdAC6VeM/b9MPSPPoA3VHtNws+hD2IrQce0TL20XPPwv8F9gWqAt0jx7fM/qAdo0+dGdG+6lfxj7HAuemxXMLcG90vw8wD2gLbA78FngtbV2PvnR+BGxRxnv7C/ByOe/7I0q+wMdHX0QdCF/mT1LyxV3RMRhP+EJvH8VYl/Brfefoy6o78A3QOVr/YEp9sVN2oriPkBT2ANYCbdPfU3TMmwMzSm8vbbu/BD6q4P//oej97BPF/ygwLO3504Am0XOXA0uBBmlxrwOOi47NFsBehMS6efRe5gCXROs3InzpXw40iJa7lj4Gaft+Chgc/Z/8mJDIU/9nA4D1wK+ifW3BponiSMIX/DbR/0NbYIe093xjhs/BlYTPwa7Ra/cAmiT9WS30W+IB6FaN/7zwAVlN+OXkwEvANtFzVwGPlFp/DOGLfwfCL+Nty9jmPcAfSz02l5JEkv6hPAcYG903wq/Xg6Ll0cDAtG1sRvjS3TFaduDQDO/t/vQvvVLPTSL6pU74sv9L2nPtCL8462Q6BmmvvaGCYzwCuDi6fzDZJYrmac+/CfSP7s8Hjkx77pzS20t77lpgUgWxPQTcn7bcC3g3w/orgD3S4p5QwfYvAZ6K7p8MTCtnve+PQbS8PSFBbpH22MnAuOj+AGBhqW0MoCRRHAq8R0ham5XxnjMlirlAnzg+b7X5lm/nZKXyjnP3RoQvsd2AptHjOwInmtnK1A04gJAkWgBfuPuKMra3I3B5qde1IJxmKe1JoJuZ7QAcREg+r6Rt5460bXxBSCbN0l6/KMP7+jyKtSw7RM+XtZ2PCC2DpmQ+BmXGYGZHmdkkM/siWr8XJcc0W0vT7n8DpAYY/LTU/jK9/+WU//6z2RdmdoWZzTGzL6P30phN30vp976LmT0TDYz4Cvhz2votCKdzsrEj4f/gk7TjPpjQsihz3+ncfSzhtNddwGdmNsTMts5y35WJU7KkRFEk3P1lwq+tW6OHFhF+TW+TdtvK3f8SPfcjM9umjE0tAv5U6nVbuvvQMva5AngBOAk4hdAC8LTt/KLUdrZw99fSN5HhLb0IdDWzFukPmllXwpfB2LSH09dpSTil8nkFx+AHMZhZfULyuxXY3t23AZ4jJLiK4s3GJ4RTTmXFXdpLQHMz61KVHZnZgYQ+kH6EluM2wJeUvBf44fu5B3gXaOPuWxPO9afWXwT8rJzdld7OIkKLomnacd/a3dtneM2mG3S/0933IrQQdyGcUqrwddG+d65gHakkJYri8negh5ntQeikPMbMjjSzOmbWIBre2dzdPyGcGrrbzLY1s7pmdlC0jfuAX5pZ12gk0FZmdrSZNSpnn48BZwAnRPdT7gWuMbP2AGbW2MxOzPaNuPuLhC/LJ82sffQe9o3e1z3u/n7a6qeZWTsz2xK4ARju7hsyHYNydlsPqA8sA9ab2VFA+pDNT4EmZtY42/dRyuOEY7KtmTUDLixvxej93Q0MjWKuF8Xf38yuzmJfjQj9AMuAzc3sOqCiX+WNCJ3Hq81sN+C8tOeeAXYws0uiYcuNoqQN4bjslBo1Fv19vQD8zcy2NrPNzGxnM+ueRdyY2d7R319d4GvCoIaNafsqL2FBOGX5RzNrE/39djSzJtnsV8qnRFFE3H0Z8G/gOndfROhQ/g3hy2IR4VdZ6v/8dMIv73cJndeXRNuYDJxLaPqvIHRID8iw21GEETpL3f3ttFieAm4GhkWnMWYCR1XyLfUFxgHPE/pi/kMYSfOrUus9QmhNLSV0tF4UxVDRMdiEu6+KXvs44b2fEr2/1PPvAkOB+dEplbJOx2VyA7AYWEBoMQ0n/PIuz0WUnIJZSTilcjzwdBb7GkM4bu8RTsetIfOpLoArCO95FeEHw39TT0THpgdwDOE4vw8cEj39RPTvcjObGt0/g5B4ZxOO5XCyO5UGIaHdF73uI8JpuFui5x4A2kXHf0QZr72N8P/3AiHpPUDoLJdqsJIzBSKFx8zGEzpSE7k6ujrM7DxCR3dWv7RFkqIWhUiOmNkOZrZ/dCpmV8JQ06eSjkukIrElCjN70Mw+M7OZ5TxvZnanmc0zsxlm1jmuWETyRD3C6J9VhM74kYR+CJG8Ftupp6hzdDXwb3fvUMbzvQjnmnsRLu66w927ll5PRESSFVuLwt0nEMbOl6cPIYm4u08CtonG44uISB5JshhXMzYdhbE4euyT0iua2SBCnRe22mqrvXbbbbecBCgixWPuXPj2W9iilo2B2n7tRzRcv5K3ff3n7r5dVbZREFUb3X0IMASgS5cuPnny5IQjEpGkDBkCjz1W8Xql1akDBxwA48fXeEj5J9WlYAb33AOffYZdf/1HVd1ckoliCZtemdo8ekxEalhVv1zz0csvh3+7V3JQcadOcMopNR9P3lmyBM47D046CU49NdwHuP76Km8yyUQxCrjQzIYROrO/jK7oFJEakkoQVf1yzUfdu4cv/EGDko4kz7jD/ffDFVfAunVwdM1NWxJbojCzoYRCdU0tzAr2e0KhMNz9XkINnV6EK3+/IcwDICJZyLaFkJ4g9OVaxD74AM49F8aNg0MOgfvug51rruRVbInC3U+u4HknTFwjIlmqbAtBCaKWeOcdmDIl/IGcc07om6hBBdGZLSLBY4/B9OlKAALMnAlTp8IZZ8Bxx8H8+dAknvqHShQiMavJjuTp00OnbK0YuSNl++47+POfw2377aFfP2jQILYkAUoUItVWUSKoyY7kWjNyR8r2xhswcCDMmgWnnQa33x6SRMyUKEQqoaykUFEi0GkiqRFLlsCBB4ZWxDPP1OiopoooUUitU51TQWUlBSUCidV778Euu0CzZvDf/8Jhh8HW2c4MWzOUKKTWSXUId+pU+dcqKUjOrFwJv/51uDZi/Hg46CA4/vhEQlGikFpJHcKS10aNCldUL10KV14Je++daDhKFJL3arr8RFVbEyI5cc458MADsPvuMHIkdOmSdERKFJL/qnOqqCwaOSR5J72IX5cusOOOcNVVUK9esnFFlCgkL2RqNejaASlqixbBL38J/fvD6aeH+3lGiUJqTE2PJkpRC0CK0saNMHhwaDls2JBYR3U2lCikxmg0kUiW3n8/9EVMmACHHx5+ZbVqlXRU5VKikBqlU0QiWZg9G2bMgAcfhAEDaryIX01TopBqST/dpNFEIhm8/Xb4kJx5JvTpE4r4bbtt0lFlZbOkA5DCNGQIHHww/OIXJf0L6ksQKcPatfC734XRTL/7HaxZEx4vkCQBalFIJZU1H4L6FkTK8frroYjfnDmhHPhtt+WkiF9NU6KQrA0ZEloQoAQhUqElS8IH5Sc/geeeg6OOSjqiKlOikKyl+iIGD1aCECnXnDnQtm0o4vf446GIX6NGSUdVLeqjkKwMGRJON3XvriQhUqYVK+Dss6FdO3jllfDYcccVfJIAtSiE7C6US/VJqLNapAxPPQXnnw/LlsE11yRexK+mKVHUcqX7HcqjPgmRcpx9NvzrX2HY37PPQufOSUdU45QoaonyWg2ploL6HUQqIb2I3777Qps2cMUVULdusnHFRImiyJU1nDWdWgoilfTRR6EZfsopYchrLfjwKFEUuVT9JSUEkWrauBHuuQeuvjq0KE48MemIckaJooilj1RS/SWRapg7NxTxmzgRjjginKvdaaeko8oZJYoiluqT0EglkWqaOxdmzYKHHgqnm/K8iF9NU6IoAuV1VKdOOel0k0gVTJsWPkRnnQXHHhuK+G2zTdJRJUIX3BWwsgrzpVORPpEqWLMGfvObcC3E9deXFPGrpUkC1KIoSCrMJxKTV18NRfzmzg0tib/9rSCL+NU0JYoCpJFMIjFYsgQOOSTUaBozJnRaC6BEUTDKmiBII5lEasDs2aE+U7Nm8OSTIVk0bJh0VHlFfRR5LNUHoQmCRGLwxRdhGtL27cPc1QDHHKMkUQa1KPJU6RpMOs0kUoOefBIuuACWL4drr4V99kk6orymRJGnNPeDSEwGDICHHw7F+55/XhO9Z0GJIg9p7geRGpZexG+//cLEQpdfDpvrKzAbsfZRmFlPM5trZvPM7Ooynm9pZuPMbJqZzTCzXnHGUyh0RbVIDVqwIIxg+ve/w/KgQXDVVUoSlRBbojCzOsBdwFFAO+BkM2tXarXfAo+7+55Af+DuuOIpBKnOa11RLVIDNmyAO++EDh1g0qSSVoVUWpwtin2Aee4+392/A4YBfUqt48DW0f3GwMcxxpP3UtdHaFSTSDXNmQMHHggXXxx+dc2aFfompEribHs1AxalLS8GupZa53rgBTP7FbAVcHhZGzKzQcAggJYtW9Z4oElLXSOh6yNEasi8eeHq6kcegVNPrXVF/Gpa0tdRnAw85O7NgV7AI2b2g5jcfYi7d3H3Ltttt13Og4ybWhIiNWDKFHjwwXD/mGNC38RppylJ1IA4WxRLgBZpy82jx9INBHoCuPvrZtYAaAp8FmNceUVzRohU07ffwh/+ALfeCi1ahF9bDRrA1ltX/FrJSpwtireANmbWyszqETqrR5VaZyFwGICZtQUaAMtijCnvaISTSDVMmAB77AE33xz6IKZNUxG/GMTWonD39WZ2ITAGqAM86O6zzOwGYLK7jwIuB+4zs0sJHdsD3Gvf0ASNcBKpgiVL4LDDQivixRfDfYlFrAOJ3f054LlSj12Xdn82sH+cMeSr0h3YIpKld96B3XcPRfyeeioU8dtqq6SjKmpJd2bXSqk6Ti+/rA5skax9/jmcfjp07FhSxK93byWJHNCliTlQeqrSVBVY1XESyYI7PPEEXHghrFgBv/89dC090l7ipESRA6VPMakSrEglnHlmuB6iSxd46aVw2klySokiR3QhnUglpBfx6949nG665BLVZ0qI+ihilF67SUSyNH8+HH44PPRQWB44EK64QkkiQUoUMVGHtUglbdgAf/97OLX01luwmb6e8oVSdA1K77RWh7VIJcyeDWefDW+8AUcfDffeC82bJx2VRJQoakAqQaSSg6YuFamkBQvggw/CB6l/f9VnyjNKFDUgNapJyUGkEt56K3xwzj03tCLmz4dGjZKOSsqgRFFDNKpJJEvffAPXXQe33w477hguomvQQEkij6m3SERyZ/z4MNT1b38LLQkV8SsIShTVlCoTLiIVWLwYevQI98eODR3WjRsnG5NkRYmiGlJDYEHDX0XK9fbb4d/mzWHkSJgxIxTyk4KhRFENqaGwGgIrUoZly8IvqE6dSprdvXrBllsmG5dUmjqzq0lzSYiU4g7DhsFFF8GXX4bZ57p1SzoqqQYlikpKv6hOc0mIlOH00+HRR0OF1wcegPbtk45IqinrU09mpvYiJddMgEpziHxv48aSQn6HHAK33QavvqokUSQqbFGY2X7A/UBDoKWZ7QH8wt3Pjzu4fKVrJkTSzJsXhrqefnoowzFwYNIRSQ3LpkVxO3AksBzA3d8GDoozqHylobAiadavh1tvDUX8pk2DevWSjkhiklUfhbsvsk1rr2yIJ5z8luqb0OkmqfVmzoSzzoLJk6FPH7j7bvjpT5OOSmKSTaJYFJ1+cjOrC1wMzIk3rPylUU4iwMKF8NFHYXRTv34q4lfkskkUvwTuAJoBS4AXgFrTP6FRTiKRN94IF88NGhSuh5g/Hxo2TDoqyYFs+ih2dfdT3X17d/+xu58GtI07sKSlZqdLTT4EGuUktdTXX8Nll4VrIf76V1i7NjyuJFFrZNOi+AfQOYvHikZ6aQ6VDpdabezYMKJp/nw47zz4y1+gfv2ko5IcKzdRmFk3YD9gOzO7LO2prYE6cQeWJJXmECEU8TvySGjVKjSrD6qVgx2FzC2KeoRrJzYH0gvFfwWcEGdQ+UCd1lJrTZsGe+4Zivg9/XT4MGyxRdJRSYLKTRTu/jLwspk95O4f5TAmEUnCp5+G+kyPPx6uKO3eHXr2TDoqyQPZ9FF8Y2a3AO2B72cYcfdDY4tKRHLHPdRmuvhiWL0abrwR9tsv6agkj2Qz6ulR4F2gFfAH4EPgrRhjSpSuvpZa55RTQvmNXXcNY8CvvRbq1k06Kskj2bQomrj7A2Z2cdrpqKJNFLr6WmqFjRvDRXJmcMQRYejrBRdAnaIepyJVlE2iWBf9+4mZHQ18DPwovpByK/2COgg/qNSRLUXtvffCkNczzggF/M46K+mIJM9lc+rpRjNrDFwOXEGoJHtJrFHlUHrZcNBFdVLE1q8PF8ztsUeYjlQjmSRLFbYo3P2Z6O6XwCEAZrZ/nEHlmsqGS9GbMSOUAJ8yBY4/Hu66C3bYIemopEBkuuCuDtCPUOPpeXefaWa9gd8AWwB75iZEEam2xYth0SJ44gno21dF/KRSMp16egA4B2gC3Glm/wFuBf7q7lklCTPraWZzzWyemV1dzjr9zGy2mc0ys8fKWicuGuEkRe211+Dee8P9VBG/E05QkpBKy3TqqQvQ0d03mlkDYCmws7svz2bDUYvkLqAHsBh4y8xGufvstHXaANcA+7v7CjP7cVXfSGWl13NSn4QUldWrwxDXf/wDdt45dFbXrw9bbZV0ZFKgMrUovnP3jQDuvgaYn22SiOwDzHP3+e7+HTAM6FNqnXOBu9x9RbSfzyqx/WpRPScpSi+8AB06hCRxwQUwdaqK+Em1ZWpR7GZmM6L7BuwcLRvg7t6xgm03AxalLS8GupZaZxcAM3uVUGjwend/vvSGzGwQMAigZcuWFew2exoGK0Vl0SI4+ujQipgwAQ44IOmIpEhkShS5mHNic6ANcDDQHJhgZru7+8r0ldx9CDAEoEuXLl7dnab6Jrp3r+6WRPLAlCmw117QogU89xwceCA0aFDx60SyVO6pJ3f/KNMti20vAVqkLTePHku3GBjl7uvcfQHwHiFxxEZ9E1I0li6FE0+ELl1KRmX06KEkITUumwvuquotoI2ZtTKzekB/YFSpdUYQWhOYWVPCqaj5McakvgkpfO7w8MPQrl0oA/7nP6uIn8QqtkTh7uuBC4ExwBzgcXefZWY3mNmx0WpjgOVmNhsYB1xZyQ7zrKWmNlWJDil4/fvDgAEhUUyfDtdcoyJ+Eqtsaj1hZlsALd19bmU27u7PAc+Veuy6tPsOXBbdYlPW1KYiBSW9iF+vXqEf4vzzYbM4TwqIBBX+lZnZMcB04PlouZOZlT6FlNfSTzeNH6/WhBSYd98N05A+8EBYPvNMuPBCJQnJmWz+0q4nXBOxEsDdpxPmpsh7Ot0kBW3dutD/sMceMHs2NGyYdERSS2VVZtzdv7RNL/uv9hDVOKVKh6cGguh0kxSc6dPDFdXTp4eyG//4B/zkJ0lHJbVUNolilpmdAtSJSm5cBLwWb1jVkyodnkoQaklIwVm6NNyefBJ+/vOko5FaLptE8SvgWmAt8BhhpNKNcQZVE1Q6XArOxImhHPj550PPnvDBB7DllklHJZJVH8Vu7n6tu+8d3X4b1X4SkZqwalXonD7wQPj732Ht2vC4koTkiWwSxd/MbI6Z/dHMOsQekUhtMmZMKOJ3991w8cUq4id5qcJE4e6HEGa2WwYMNrN3zOy3sUcmUuwWLYLevUPLYeLE0JrQyCbJQ1kNxHb3pe5+J/BLwjUV11XwEhEpizu8+Wa436IFjB4N06apBIfktWwuuGtrZteb2TvAPwgjnprHHplIsfnkkzANadeuJWO3Dz9cRfwk72XToniQcLHdke5+sLvfk8sJhipL05tK3nGHf/0r1GYaPRpuvhn23z/pqESyVuHwWHfvlotAakqqXIcusJO80a8fDB8eRjXdfz/sskvSEYlUSrmJwswed/d+0Smn9Cuxs53hLjEq1yGJ27AhFPDbbDM45hg49NBQmVL1maQAZWpRXBz92zsXgYgUjTlzYODAUILj3HPhjDOSjkikWjLNcPdJdPf8Mma3Oz834YkUkHXr4MYbQ1mAuXOhceOkIxKpEdm0g3uU8dhRNR1ITVBHtiRm2rQwJenvfgfHHx9aFf36JR2VSI3I1EdxHqHl8DMzm5H2VCPg1bgDqwp1ZEtiPv0UPv8cRoyAPn2SjkakRmXqo3gMGA3cBFyd9vgqd/8i1qiqQR3ZkjMTJsA778AFF4QifvPmwRZbJB2VSI3LdOrJ3f1D4AJgVdoNM/tR/KGJ5KmvvgoVXrt3hzvvLCnipyQhRaqiFkVvYApheGz6zEUO/CzGuETy03PPhWGuH38Ml10GN9ygIn5S9MpNFO7eO/q3IKY9FYndokWh/2HXXcMFdF27Jh2RSE5kU+tpfzPbKrp/mpndZmYt4w9NJA+4w6RJ4X6LFvDCC6EUuJKE1CLZDI+9B/jGzPYALgc+AB6JNSqRfPDxx3DccdCtW8m460MOgXr1ko1LJMeySRTr3d2BPsA/3f0uwhBZkeLkHmoytWsXWhC33qoiflKrZTNn9iozuwY4HTjQzDYD6sYblkiCTjgB/ve/MKrp/vuhdeukIxJJVDYtipOAtcDZ7r6UMBfFLbFGJZJrGzbAxo3h/nHHwb33wtixShIiZDcV6lLgUaCxmfUG1rj7v2OPTCRXZs4Mp5YeeCAsn366Kr2KpMlm1FM/4E3gRKAf8IaZnRB3YCKx++47+MMfoHNn+OAD2HbbpCMSyUvZ9FFcC+ydmtXOzLYDXgSGxxmYSKymTIEBA0Jr4pRT4O9/h+22SzoqkbyUTaLYrNTUp8vJrm9DJH8tXw4rV8LTT0NvTbkikkk2ieJ5MxsDDI2WTwKeiy8kkZiMGxeK+F10ERxxBLz/PjRokHRUInkvm87sK4HBQMfoNsTdr4o7sMrSXBRSri+/DJ3Thx4K99xTUsRPSUIkK5nmo2gD3ArsDLwDXOHuS3IVWGVpLgop09NPwy9/CUuXwhVXhM5rFfETqZRMLYoHgWeAvoQKsv/ISUTVoLkoZBOLFkHfvtCkSajXdMstsOWWSUclUnAy9VE0cvf7ovtzzWxqLgISqRZ3eP112G+/kiJ+++2n+kwi1ZCpRdHAzPY0s85m1hnYotRyhcysp5nNNbN5ZnZ1hvX6mpmbWZfKvgGR7y1eDMceGy6eS3VYHXywkoRINWVqUXwC3Ja2vDRt2YFDM23YzOoAdwE9gMXAW2Y2yt1nl1qvEXAx8EblQi+R6sju3r2qW5CCtnEj3HcfXHklrF8Pt90GBxyQdFQiRSPTxEWHVHPb+wDz3H0+gJkNI1SgnV1qvT8CNwNXVnYHQ4aETuzUj0d1ZNdSffvCiBFhVNN998HPNPmiSE2K88K5ZsCitOXF0WPfi05htXD3ZzNtyMwGmdlkM5u8bNmy7x9/7DGYPj20JAYPVkd2rbJ+fUkRv759Q4J48UUlCZEYZHPBXSyicuW3AQMqWtfdhwBDALp06eLpz3XqBOPHxxCg5K8ZM2DgQDjnnHB9xGmnJR2RSFGLs0WxBGiRttw8eiylEdABGG9mHwL7AqPUoS3lWrsWfv972Gsv+Ogj1WYSyZFsqsdaNFf2ddFySzPbJ4ttvwW0MbNWZlYP6A+MSj3p7l+6e1N338nddwImAce6++QqvRMpbm+9Faq83nADnHwyzJkDP/950lGJ1ArZtCjuBroBJ0fLqwijmTJy9/XAhcAYYA7wuLvPMrMbzOzYKsYrtdWKFbB6NTz3HPz73+EiOhHJiWz6KLq6e2czmwbg7iuiFkKF3P05ShUQdPfryln34Gy2KbXI2LGhiN/FF4cifu+9p/IbIgnIpkWxLromwuH7+Sg2xhpVFlQEsIitXAnnnguHHRaGs6WK+ClJiCQim0RxJ/AU8GMz+xMwEfhzrFFlQUUAi9TIkdCuHTz4IPz612GCISUIkURVeOrJ3R81synAYYABx7n7nNgjy4KKABaZhQvhxBOhbVsYNQq6aACcSD6oMFGYWUvgG+Dp9MfcfWGcgUkt4Q4TJ8KBB0LLluGiuX33VX0mkTySTWf2s4T+CQMaAK2AuUD7GOOS2mDhwjBXxOjR4arJ7t3hoIOSjkpESsnm1NPu6ctR2Y3zY4tIit/GjXDvvXDVVaFFceedKuInkscqXcLD3SeLxI4AABSfSURBVKeaWdc4gpFa4uc/D53WPXqE4Ws77ZR0RCKSQTZ9FJelLW4GdAY+ji0iKU7r18Nmm4XbSSdBnz4wYACYJR2ZiFQgm+GxjdJu9Ql9Fn3iDEqKzNtvQ9euofUAoQTHWWcpSYgUiIwtiuhCu0bufkWO4pFismYN3Hgj3Hwz/OhH8JOfJB2RiFRBuS0KM9vc3TcA++cwnqzoquwC8OabsOee8Kc/wamnhiJ+xx2XdFQiUgWZWhRvEvojppvZKOAJ4OvUk+7+v5hjK5euyi4AX30F334Lzz8PRx6ZdDQiUg3ZjHpqACwnzJGdup7CgcQSBeiq7Lz0wgswaxZceikcfjjMnavyGyJFIFOi+HE04mkmJQkixct+idRKK1bAZZfBQw9B+/Zw/vkhQShJiBSFTKOe6gANo1ujtPupmwj873+hiN8jj8A118DkyUoQIkUmU4viE3e/IWeRSOFZuBD694cOHcKEQnvumXREIhKDTC0KDXKXH3IvGXLWsmWYXOiNN5QkRIpYpkRxWM6ikMLw0Udw1FFw8MElyeKAA6Bu3UTDEpF4lZso3P2LXAYieWzjRvjnP0NH9cSJ8I9/hLLgIlIrVLoooNRCxx0HTz8drocYPBh23DHpiEQkh5QopGzr1kGdOqGI38knwwknwOmnqz6TSC2UTVFAqW2mToV99glzRkBIFGecoSQhUksVXKJYtkx1nmLz7bfhWoh99oGlS6FFi6QjEpE8UHCnnr6IuthV56mGTZoEZ54J770HZ58Nt94K226bdFQikgcKLlGA6jzF4uuvQ7/E//1fqNMkIhIpyEQhNeT550MRv8svh8MOg3ffhXr1ko5KRPJMwfVRSA1YvjycZjrqKHj4Yfjuu/C4koSIlEGJojZxh+HDQxG/xx6D3/4W3npLCUJEMtKpp9pk4cIwCqBjxzB3xB57JB2RiBSAgmtRrF6ddAQFxj0U7oNwRfX48WGEk5KEiGSp4BIFaGhs1hYsgCOOCB3VqYtP9tsPNldDUkSyV3CJomFDDY2t0IYNcMcdYZ6IN96Ae+5RET8RqTL9tCxGffrAs89Cr16hDIeusBaRalCiKBbpRfxOPz3UZzrlFNVnEpFqi/XUk5n1NLO5ZjbPzK4u4/nLzGy2mc0ws5fMTPWrq2LyZOjSJZxiAjjpJDj1VCUJEakRsSUKM6sD3AUcBbQDTjazdqVWmwZ0cfeOwHDgr3HFU5S+/Rauugq6dg3VEjVPhIjEIM4WxT7APHef7+7fAcOAPukruPs4d/8mWpwENI8xnuLy+uthiOtf/xqK+M2eDb17Jx2ViBShOPsomgGL0pYXA10zrD8QGF3WE2Y2CBgEUL9+x5qKr7B9+22YovTFF8PwVxGRmORFZ7aZnQZ0AbqX9by7DwGGADRq1MVzGFp+ee65UMTvyivh0ENhzhyoWzfpqESkyMV56mkJkD4us3n02CbM7HDgWuBYd18bYzyF6/PP4bTT4Oij4dFHS4r4KUmISA7EmSjeAtqYWSszqwf0B0alr2BmewKDCUnisxhjKUzuMGwYtG0Ljz8Ov/89vPmmiviJSE7FdurJ3deb2YXAGKAO8KC7zzKzG4DJ7j4KuAVoCDxhYSjnQnc/Nq6YCs7ChaEc+B57wAMPwO67Jx2RiNRC5l5Yp/wbNeriq1ZNTjqM+LjDSy+VzDI3aRLsvXe4mE5EpIrMbIq7d6nKawuu1lNR++CDMIKpR4+SIn777qskISKJUqLIBxs2wG23hVNLU6bA4MEq4icieSMvhsfWesccA6NHhwvm7rkHmuu6QxHJH0oUSfnuuzAvxGabwYABoZBf//6qzyQieUennpLw5puw115w991huV+/UO1VSUJE8pASRS598w1cfjl06wYrVsDOOycdkYhIhXTqKVcmTgzXRMyfD7/4Bdx8MzRunHRUIiIVUqLIldTEQuPGwcEHJx2NiEjWlCji9PTToXDfr38NhxwSSoFvrkMuIoVFfRRxWLYsTEN67LEwdGhJET8lCREpQEoUNckdHnssFPEbPhxuuAHeeENF/ESkoOknbk1auBDOOgv23DMU8WvfPumIRESqTS2K6tq4EcaMCfd33BFeeQVefVVJQkSKhhJFdbz/fphprmdPmDAhPLbPPiriJyJFRYmiKtavh1tugY4dYfr0cJpJRfxEpEipj6IqevcOp5v69AllOH7606QjEslL69atY/HixaxZsybpUGqNBg0a0Lx5c+rW4FTJmrgoW2vXhjmqN9ssjGjauBFOPFH1mUQyWLBgAY0aNaJJkyaYPiuxc3eWL1/OqlWraNWq1SbPaeKiuE2aBJ07w113heUTTgiF/PSHL5LRmjVrlCRyyMxo0qRJjbfglCgy+fpruPRS2G8/WLUK2rRJOiKRgqMkkVtxHG/1UZTnlVdCEb8FC+D88+Gmm2DrrZOOSkQk59SiKM/69aFP4uWXwyknJQmRgjVixAjMjHfffff7x8aPH0/v3r03WW/AgAEMHz4cCB3xV199NW3atKFz585069aN0aNHVzuWm266idatW7PrrrsyJnUNViljx46lc+fOdOjQgTPPPJP169cDoQ/ioosuonXr1nTs2JGpU6dWO55sKFGkGzEitBwgFPGbNQsOOijZmESk2oYOHcoBBxzA0KFDs37N7373Oz755BNmzpzJ1KlTGTFiBKtWrapWHLNnz2bYsGHMmjWL559/nvPPP58NGzZsss7GjRs588wzGTZsGDNnzmTHHXfk4YcfBmD06NG8//77vP/++wwZMoTzzjuvWvFkS6eeAD79FH71K3jiidBpffnloT6TiviJ1JhLLgmXHdWkTp3g73/PvM7q1auZOHEi48aN45hjjuEPf/hDhdv95ptvuO+++1iwYAH169cHYPvtt6dfv37VinfkyJH079+f+vXr06pVK1q3bs2bb75Jt27dvl9n+fLl1KtXj1122QWAHj16cNNNNzFw4EBGjhzJGWecgZmx7777snLlSj755BN22GGHasVVkdrdonCHRx6Bdu1g5Ej405/CCCcV8RMpGiNHjqRnz57ssssuNGnShClTplT4mnnz5tGyZUu2zuKU86WXXkqnTp1+cPvLX/7yg3WXLFlCixYtvl9u3rw5S5Ys2WSdpk2bsn79eiZPDpcBDB8+nEWLFmX9+jjU7p/MCxfCOedAly7h6urddks6IpGiVdEv/7gMHTqUiy++GID+/fszdOhQ9tprr3JHB1V21NDtt99e7RhL73/YsGFceumlrF27liOOOII6CZcFqn2JIlXE76ijQhG/V18N1V5Vn0mk6HzxxReMHTuWd955BzNjw4YNmBm33HILTZo0YcWKFT9Yv2nTprRu3ZqFCxfy1VdfVdiquPTSSxk3btwPHu/fvz9XX331Jo81a9bs+9YBwOLFi2nWrNkPXtutWzdeeeUVAF544QXee++9Sr2+xrl7Qd0aNtzLq2zuXPcDD3QH9/Hjq74dEcnK7NmzE93/4MGDfdCgQZs8dtBBB/nLL7/sa9as8Z122un7GD/88ENv2bKlr1y50t3dr7zySh8wYICvXbvW3d0/++wzf/zxx6sVz8yZM71jx46+Zs0anz9/vrdq1crXr1//g/U+/fRTd3dfs2aNH3roof7SSy+5u/szzzzjPXv29I0bN/rrr7/ue++9d5n7Keu4A5O9it+7taOPYv16uPnmUMTvnXfgX//SaCaRWmDo0KEcf/zxmzzWt29fhg4dSv369fnPf/7DWWedRadOnTjhhBO4//77ady4MQA33ngj2223He3ataNDhw707t07qz6LTNq3b0+/fv1o164dPXv25K677vr+tFKvXr34+OOPAbjlllto27YtHTt25JhjjuHQQw/9fp2f/exntG7dmnPPPZe77767WvFkq3bUejrySHjhBfj5z8M1ET/5STzBicgm5syZQ9u2bZMOo9Yp67hXp9ZT8fZRrFkTLpirUwcGDQq3vn2TjkpEpOAU56mnV18NA6xTRfz69lWSEBGpouJKFKtXw0UXhUmE1qwBNXlFEldop7cLXRzHu3gSxcsvQ4cO8M9/woUXwsyZ0KNH0lGJ1GoNGjRg+fLlShY54tF8FA0aNKjR7RZXH8WWW4aqr/vvn3QkIkK4cnjx4sUsW7Ys6VBqjdQMdzWpsEc9/e9/8O678JvfhOUNG3ThnIhIGfJ2hjsz62lmc81snpldXcbz9c3sv9Hzb5jZTllteOnSMMtc377w1FPw3XfhcSUJEZEaF1uiMLM6wF3AUUA74GQza1dqtYHACndvDdwO3FzRdhuvWx46qZ95JpQEf+01FfETEYlRnC2KfYB57j7f3b8DhgF9Sq3TB3g4uj8cOMwqqMi1/dqPQqf122/D1VeHayVERCQ2cXZmNwMWpS0vBrqWt467rzezL4EmwOfpK5nZIGBQtLjWJk6cqUqvADSl1LGqxXQsSuhYlNCxKLFrVV9YEKOe3H0IMATAzCZXtUOm2OhYlNCxKKFjUULHooSZVbL2UYk4Tz0tAVqkLTePHitzHTPbHGgMLI8xJhERqaQ4E8VbQBsza2Vm9YD+wKhS64wCzozunwCM9UIbrysiUuRiO/UU9TlcCIwB6gAPuvssM7uBUBd9FPAA8IiZzQO+ICSTigyJK+YCpGNRQseihI5FCR2LElU+FgV3wZ2IiORW8dR6EhGRWChRiIhIRnmbKGIr/1GAsjgWl5nZbDObYWYvmdmOScSZCxUdi7T1+pqZm1nRDo3M5liYWb/ob2OWmT2W6xhzJYvPSEszG2dm06LPSa8k4oybmT1oZp+Z2cxynjczuzM6TjPMrHNWG67qZNtx3gid3x8APwPqAW8D7Uqtcz5wb3S/P/DfpONO8FgcAmwZ3T+vNh+LaL1GwARgEtAl6bgT/LtoA0wDto2Wf5x03AkeiyHAedH9dsCHSccd07E4COgMzCzn+V7AaMCAfYE3stluvrYoYin/UaAqPBbuPs7dv4kWJxGuWSlG2fxdAPyRUDdsTS6Dy7FsjsW5wF3uvgLA3T/LcYy5ks2xcGDr6H5j4OMcxpcz7j6BMIK0PH2Af3swCdjGzHaoaLv5mijKKv/RrLx13H09kCr/UWyyORbpBhJ+MRSjCo9F1JRu4e7P5jKwBGTzd7ELsIuZvWpmk8ysZ86iy61sjsX1wGlmthh4DvhVbkLLO5X9PgEKpISHZMfMTgO6AN2TjiUJZrYZcBswIOFQ8sXmhNNPBxNamRPMbHd3X5loVMk4GXjI3f9mZt0I1291cPeNSQdWCPK1RaHyHyWyORaY2eHAtcCx7r42R7HlWkXHohHQARhvZh8SzsGOKtIO7Wz+LhYDo9x9nbsvAN4jJI5ik82xGAg8DuDurwMNCAUDa5usvk9Ky9dEofIfJSo8Fma2JzCYkCSK9Tw0VHAs3P1Ld2/q7ju5+06E/ppj3b3KxdDyWDafkRGE1gRm1pRwKmp+LoPMkWyOxULgMAAza0tIFLVxftZRwBnR6Kd9gS/d/ZOKXpSXp548vvIfBSfLY3EL0BB4IurPX+juxyYWdEyyPBa1QpbHYgxwhJnNBjYAV7p70bW6szwWlwP3mdmlhI7tAcX4w9LMhhJ+HDSN+mN+D9QFcPd7Cf0zvYB5wDfAWVlttwiPlYiI1KB8PfUkIiJ5QolCREQyUqIQEZGMlChERCQjJQoREclIiULykpltMLPpabedMqy7ugb295CZLYj2NTW6erey27jfzNpF939T6rnXqhtjtJ3UcZlpZk+b2TYVrN+pWCulSu5oeKzkJTNb7e4Na3rdDNt4CHjG3Yeb2RHAre7esRrbq3ZMFW3XzB4G3nP3P2VYfwChgu6FNR2L1B5qUUhBMLOG0VwbU83sHTP7QdVYM9vBzCak/eI+MHr8CDN7PXrtE2ZW0Rf4BKB19NrLom3NNLNLose2MrNnzezt6PGTosfHm1kXM/sLsEUUx6PRc6ujf4eZ2dFpMT9kZieYWR0zu8XM3ormCfhFFofldaKCbma2T/Qep5nZa2a2a3SV8g3ASVEsJ0WxP2hmb0brllV9V2RTSddP1023sm6EK4mnR7enCFUEto6ea0q4sjTVIl4d/Xs5cG10vw6h9lNTwhf/VtHjVwHXlbG/h4ATovsnAm8AewHvAFsRrnyfBewJ9AXuS3tt4+jf8UTzX6RiSlsnFePxwMPR/XqESp5bAIOA30aP1wcmA63KiHN12vt7AugZLW8NbB7dPxx4Mro/APhn2uv/DJwW3d+GUP9pq6T/v3XL71telvAQAb51906pBTOrC/zZzA4CNhJ+SW8PLE17zVvAg9G6I9x9upl1J0xU82pU3qQe4Zd4WW4xs98SagANJNQGesrdv45i+B9wIPA88Dczu5lwuuqVSryv0cAdZlYf6AlMcPdvo9NdHc3shGi9xoQCfgtKvX4LM5sevf85wP+lrf+wmbUhlKioW87+jwCONbMrouUGQMtoWyJlUqKQQnEqsB2wl7uvs1AdtkH6Cu4+IUokRwMPmdltwArg/9z95Cz2caW7D08tmNlhZa3k7u9ZmPeiF3Cjmb3k7jdk8ybcfY2ZjQeOBE4iTLIDYcaxX7n7mAo28a27dzKzLQm1jS4A7iRM1jTO3Y+POv7Hl/N6A/q6+9xs4hUB9VFI4WgMfBYliUOAH8wLbmGu8E/d/T7gfsKUkJOA/c0s1eewlZntkuU+XwGOM7MtzWwrwmmjV8zsp8A37v4fQkHGsuYdXhe1bMryX0IxtlTrBMKX/nmp15jZLtE+y+RhRsOLgMutpMx+qlz0gLRVVxFOwaWMAX5lUfPKQuVhkYyUKKRQPAp0MbN3gDOAd8tY52DgbTObRvi1foe7LyN8cQ41sxmE0067ZbNDd59K6Lt4k9Bncb+7TwN2B96MTgH9HrixjJcPAWakOrNLeYEwudSLHqbuhJDYZgNTzWwmoWx8xhZ/FMsMwqQ8fwVuit57+uvGAe1SndmElkfdKLZZ0bJIRhoeKyIiGalFISIiGSlRiIhIRkoUIiKSkRKFiIhkpEQhIiIZKVGIiEhGShQiIpLR/wMMoZ3GcWYzAwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Compute predicted probabilities\n",
        "nb_model = MultinomialNB(alpha=1.8)\n",
        "nb_model.fit(X_train_tfidf, y_train)\n",
        "probs = nb_model.predict_proba(X_val_tfidf)\n",
        "\n",
        "# Evaluate the classifier\n",
        "evaluate_roc(probs, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 327
        },
        "id": "MKMBotb3lbfB",
        "outputId": "53155806-9c82-4a9b-b1d3-19917cce2562"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AUC: 0.5743\n",
            "Accuracy: 50.00%\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5gUZfLA8W+B5KSidxIFBVQQROBEVEDFgIiiggQjCGIWEyeKZ0AMHJh/JlAPTKBiAgXxVMIZQMlREEFhQSQjWVjq90f1usOyOzsbemZ2tz7PM89Oh+mu6d2dmu6333pFVXHOOeeyUizRATjnnEtuniicc85F5YnCOedcVJ4onHPOReWJwjnnXFSeKJxzzkXlicLliIgsEJHTEx1HshCRe0XklQTte7iIDEzEvvObiFwuIp/n8rX+NxkyTxQFmIj8IiI7RWSbiKwJPjjKh7lPVW2gqpPC3EcaESklIo+JyIrgff4kIn1FROKx/0ziOV1EUiLnqeqjqtorpP2JiNwqIvNFZLuIpIjIeyLSMIz95ZaIPCgib+ZlG6r6lqqeE8O+DkiO8fybLKo8URR8F6hqeaAxcCJwT4LjyTEROSiLRe8BbYB2QAXgSqA38EwIMYiIJNv/wzNAH+BW4FCgHvARcH5+7yjK7yB0idy3i5Gq+qOAPoBfgLMipv8NfBoxfTLwLbAZmAOcHrHsUOA/wGpgE/BRxLL2wOzgdd8CjTLuE6gK7AQOjVh2IrAeKBFMXwMsCrY/ATgyYl0FbgJ+ApZn8t7aALuAGhnmNwdSgTrB9CTgMeB74A/g4wwxRTsGk4BHgG+C91IH6BHEvBVYBlwXrFsuWGcfsC14VAUeBN4M1qkVvK+rgRXBsegfsb8ywIjgeCwC/gmkZPG7rRu8z5Oi/P6HA88DnwbxTgOOjlj+DLAyOC4zgJYRyx4ERgNvBst7AScB3wXH6jfg/4CSEa9pAPwX2Aj8DtwLtAX+BPYEx2ROsG4l4NVgO6uAgUDxYFn34Jg/BWwIlnUHvg6WS7BsbRDbPOB47EvCnmB/24CxGf8PgOJBXD8Hx2QGGf6G/JGLz5pEB+CPPPzy9v8HqR78Qz0TTFcL/gnbYWeOZwfThwfLPwXeAQ4BSgCtg/knBv+gzYN/uquD/ZTKZJ9fAddGxDMYeCl43gFYChwHHATcB3wbsa4GHzqHAmUyeW+PA5OzeN+/kv4BPin4IDoe+zB/n/QP7uyOwSTsA71BEGMJ7Nv60cGHVWtgB9AkWP90Mnywk3miGIYlhROA3cBxke8pOObVgbkZtxex3euBX7P5/Q8P3s9JQfxvAaMill8BVA6W3QmsAUpHxL0HuCg4NmWAplhiPSh4L4uA24L1K2Af+ncCpYPp5hmPQcS+PwReDn4nf8MSedrvrDuwF7gl2FcZ9k8U52If8AcHv4fjgCoR73lglP+Dvtj/wTHBa08AKif6f7WgPxIegD/y8Muzf5Bt2DcnBb4EDg6W3Q28kWH9CdgHfxXsm/EhmWzzReDhDPMWk55IIv8pewFfBc8F+/baKpgeD/SM2EYx7EP3yGBagTOjvLdXIj/0MiybSvBNHfuwfzxiWX3sG2fxaMcg4rUDsjnGHwF9guenE1uiqB6x/Huga/B8GXBuxLJeGbcXsaw/MDWb2IYDr0RMtwN+jLL+JuCEiLinZLP924APg+fdgFlZrPfXMQim/44lyDIR87oBE4Pn3YEVGbbRnfREcSawBEtaxTJ5z9ESxWKgQxj/b0X5kWzXZF3OXaSqFbAPsWOBw4L5RwKXisjmtAdwGpYkagAbVXVTJts7Ergzw+tqYJdZMnofaCEiVYBWWPL5X8R2nonYxkYsmVSLeP3KKO9rfRBrZqoEyzPbzq/YmcFhRD8GmcYgIueJyFQR2Ris3470YxqrNRHPdwBpNxhUzbC/aO9/A1m//1j2hYjcJSKLRGRL8F4qsf97yfje64nIJ8GNEX8Aj0asXwO7nBOLI7HfwW8Rx/1l7Mwi031HUtWvsMtezwNrRWSoiFSMcd85idPFyBNFIaGqk7FvW0OCWSuxb9MHRzzKqerjwbJDReTgTDa1Engkw+vKqurITPa5Cfgc6AJchp0BaMR2rsuwnTKq+m3kJqK8pS+A5iJSI3KmiDTHPgy+ipgduU5N7JLK+myOwQExiEgpLPkNAf6uqgcD47AEl128sfgNu+SUWdwZfQlUF5FmudmRiLTE2kA6Y2eOBwNbSH8vcOD7eRH4EairqhWxa/1p668Ejspidxm3sxI7ozgs4rhXVNUGUV6z/wZVn1XVptgZYj3sklK2rwv2fXQ267gc8kRRuDwNnC0iJ2CNlBeIyLkiUlxESge3d1ZX1d+wS0MviMghIlJCRFoF2xgGXC8izYM7gcqJyPkiUiGLfb4NXAV0Cp6neQm4R0QaAIhIJRG5NNY3oqpfYB+W74tIg+A9nBy8rxdV9aeI1a8QkfoiUhYYAIxW1dRoxyCL3ZYESgHrgL0ich4Qecvm70BlEakU6/vI4F3smBwiItWAm7NaMXh/LwAjg5hLBvF3FZF+MeyrAtYOsA44SETuB7L7Vl4BazzeJiLHAjdELPsEqCIitwW3LVcIkjbYcamVdtdY8Pf1OfCEiFQUkWIicrSItI4hbkTkH8HfXwlgO3ZTw76IfWWVsMAuWT4sInWDv99GIlI5lv26rHmiKERUdR3wOnC/qq7EGpTvxT4sVmLfytJ+51di37x/xBqvbwu2MR24Fjv134Q1SHePstsx2B06a1R1TkQsHwKDgFHBZYz5wHk5fEsdgYnAZ1hbzJvYnTS3ZFjvDexsag3W0HprEEN2x2A/qro1eO272Hu/LHh/act/BEYCy4JLKpldjotmAJACLMfOmEZj37yzcivpl2A2Y5dULgbGxrCvCdhxW4JdjttF9EtdAHdh73kr9oXhnbQFwbE5G7gAO84/AWcEi98Lfm4QkZnB86uwxLsQO5ajie1SGlhCGxa87lfsMtzgYNmrQP3g+H+UyWufxH5/n2NJ71WssdzlgaRfKXCu4BGRSVhDakJ6R+eFiNyANXTH9E3buUTxMwrn4kREqojIqcGlmGOwW00/THRczmUntEQhIq+JyFoRmZ/FchGRZ0VkqYjMFZEmYcXiXJIoid39sxVrjP8Ya4dwLqmFdukpaBzdBryuqsdnsrwddq25Hda56xlVbZ5xPeecc4kV2hmFqk7B7p3PSgcsiaiqTgUODu7Hd845l0QSWYyrGvvfhZESzPst44oi0hur80K5cuWaHnvssXEJ0Dnnkt2ePbB9e/rzFSv2X16TXzmYzcxl73pVPTw3+ygQVRtVdSgwFKBZs2Y6ffr0BEfknHOJtWcPzJwJ//wnzJ27/7LR7ynNmwMilHv9RYptWMvBTz74a273lchEsYr9e6ZWD+Y555zLxrPPwl132fNTT4XnnrPnFbeu4qghNyC7u8Dll8O9Qb/JJx/M9b4SeXvsGOCq4O6nk4EtQY9O55xz2fjjD/s5YQKMHg0nNlZOnD6Moy+oj3zxBWzblm/7Cu2MQkRGYoXqDhMbFewBrFAYqvoSVkOnHdbzdwc2DoBzzrkcOOcc4Oefoc21MHEinHEGDBsGR+dfyavQEoWqdstmedrANc4552K0eDE0b25nFMXSrgnNmwczZsDQodCrF+TzaMEFojHbOeec+fVX2LIF7rlgPu2OmAlcBRddBMuWQeVw6h96onDOuQJE9vzJAzzK/eMfpdgRf4ddnaF06dCSBHitJ+ecKxBU4dEO06jdqQkP8hDr23SBWbMsSYTME4VzziW5ESPguvaruGtMS8ru2cIDTT+hxMg34LCcDr6YO37pyTnnktTs2fDJk0t4elw9du2qRunq79B1WBseahvryLD5wxOFc84lo82b2XbZP7l30St8V34SLe9rRb9+FyckFE8UzjmXbMaMgRtu4JTf1vB82b58uvYfCR2nz9sonHMumfTqBR06QOXKDLp4GgPLD4IyiR3N1c8onHMugbZsgdmzgnGBRKhSsRklrjmSld3uZupzJRMbXMAThXPOJdBDvVbSZvT1jKIrb3IlcL0teM1+5GMljlzzROGcc3G2bRvs3L6P0iNeZuBHd4OkcvQdF3PN+Qeue9RR8Y8vI08UzjkXR8uWwfn1fuKl1F60Zgr/5SyeqT+UT4bUJlmHZPPGbOeci4O5c6F2bbuUVC91If8oNZcvL3+NJc99zgPDayc6vKj8jMI550L2v//BPe3mcFGJ2VS8/2rKl++AXL6MNlUPoU2ig4uBJwrnnAvRpx/sZm7ngUxMfRypWoWD7ukS1Gc6JNGhxcwvPTnnXD5btQrOOw9uaPwdtTueyD2pA0ntfBkHzY1PEb/85mcUzjmXz2bMgLmfrWKMtOaPckewY8Q4ynY8L9Fh5ZqfUTjnXH5atAiA1VRjxeB3qfzbggKdJMDPKJxzLk8WLIB33oHSOzfR9r930mTOf5hxxhSgJVvOuAgqJDrCvPNE4ZxzefD007D+lQ95gRs5nHU8xj38e+I/OOQQqFo10dHlD08UzjkXo1mzYPHi/edd/Mk1tOM/0LgxvPop9zRpwj2JCS80niiccy4bGzbYIEJdu8L69QBBET+EazmZn6vU5Zbv74ISJRIYZXg8UTjnXCaWLYOtW+35PffA+PH2/L4rf+Xu5dfxx/mX8cdFVwG9qVYNKJw5AvBE4Zxz+9m+HaZNgzYZukyfeMI+3jvzRY4a1g9RpXz3S6marMWZ8pknCudckbdnj/3ct8+qta5da9OPPw716kH5VYtpOaIXpZ/6Gs45B15+GWrVSli88eaJwjlXpD3/PNx88/7zLrkEOnWCSy+Fgw4CxiyGnxfA8OFw1VUgkohQE8YThXOuSJozB9q2hY0bbaTR/v1tfvHicPXVUGXNLHhjNvToARdeaI0WBx+c2KATxBOFc67QmjoVrrkm/dJSpK1b4fff4Yor4KyzLDkAsGsXDBgA//43VKsG3bpZfaYimiTAE4VzrhCbMcMqalxySea1+A49FJ58MuKu1m++gZ49rbNEjx7wxBMFsohffvNE4Zwr9F56CQ4/PJuVVq2CM86ws4gJE6zR2gFeFNA5V9QtXGg/q1WD99+HefM8SWTgZxTOuaS3cCG8+SaoZr9upBkzoizcuBHuuANGjIDJk6FVK7jggjzFWVh5onDOJa1t2+D11+Htt635oGTJnG/jqKOgYsUMM99/H266yWpz9O8PJ52UL/EWVp4onHNJ65NP7PMc7LN82rR82Gj37nYW0aQJfPaZFfNzUXmicM4lrb177efMmdCwYR42lHbNSgROOQWOOw7uvDPoTeeyE2pjtoi0FZHFIrJURPplsrymiEwUkVkiMldE2oUZj3OuYKpQIQ+f6cuXW+P066/bdO/ecPfdniRyILQjJSLFgeeBs4EU4AcRGaOqCyNWuw94V1VfFJH6wDigVlgxOefyZu9eWL06fvuzkt65lJpq9TnuuQeKFYPLL8+3uIqaMFPqScBSVV0GICKjgA5AZKJQIK2ZqRIQxz9B51ws9uxJvwR0441W7ijeSpXK4QsWLbKOc999B+edZx0patYMJbaiIMxEUQ1YGTGdAjTPsM6DwOcicgtQDjgrsw2JSG+gN0BN/2U7Fzc//QSNGllVizRHHw333hu/GA4/HGrUyOGLli613tVvvGFnEkWsiF9+S/RFum7AcFV9QkRaAG+IyPGqui9yJVUdCgwFaNasWQ7vpHbO5dbq1ZYkrr3WEgRYd4MWLRIbV6ZmzLBKf9dcY/0hli/P5L5YlxthJopVQOT3gOrBvEg9gbYAqvqdiJQGDgPWhhiXcy6Ke++FkSPt+c6d9rNbN6tukZR27oSHHoIhQ+zU47LLrD6TJ4l8E2ai+AGoKyK1sQTRFbgswzorgDbAcBE5DigNrAsxJueKtLffhqeeir7OwoVWKPWs4EJwhQrQrFn4seXKlCnQq5ddI+vZ05KFF/HLd6ElClXdKyI3AxOA4sBrqrpARAYA01V1DHAnMExEbscatrur5rSTvnMuVp9+CgsWRD87+NvfbGyeLl3iF1eurFpl45XWqAFffHHg2KUu34TaRqGq47BbXiPn3R/xfCFwapgxOOf2V62aJYwCa948631XrRp8+KFlvXLlEh1VoZboxmznXIj27oVBg2DTJpueOTOx8eTJ+vVw++1WHTCtiF/79omOqkjwROFcITV2LHz1FTz9tPVDSOuIXOAKpKrCe+/ZwNabNsEDD0DzjHfauzB5onCukNi40ZJDaqpN33yz3RBUooRVXm3aNLHx5drVV1t/iGbN4Msv81j0yeWGJwrnCokXX4T77tt/3uOP29Wa3JTnTqjIIn6tW1uvv9tu8/pMCeJH3blC4s8/7eevv9rPYsWsvbfAdUpetsx6+F1xhY1b3bNnoiMq8nwoVOcKgd9+s0tPYCWNataE6tULWJJITbUGlYYN4YcfLNO5pOBnFM4VcDNnprc/5Lh4XrJYuNBKb0ybBuefb0X8qldPdFQu4InCuQIurRT3v/4F556b2Fhybfly+Pln6zretWsBOxUq/DxROFdItG1rg7cVGD/8ALNnW3vE+edb20SFComOymXCLwI65+Jrxw646y44+WR47LH0GuaeJJKWJwrnXPxMmmS3uj7xhJ1JzJrlRfwKAL/05JyLj5QUOPtsOPJI6zKetHXLXUZ+RuGcC9ecOfazenX4+GOYO9eTRAHjicI5F45162wQocaNrYgfQLt2ULZsYuNyOeaXnpxz+UsVRo2CW2+FLVts9LmkHDvVxcoThXMF0KxZ8P339nzhwsTGcoArr4S33rIKr6++Cg0aJDoil0cxJwoRKauqO8IMxjkXm969Yfr09OnixW1kuoTZt886yYlY+0PTpnZGUbx4AoNy+SXbNgoROUVEFgI/BtMniMgLoUfmnMvSn39aB7vVq+2xYQPUqZOgYJYutWFI//Mfm+7Z00rWepIoNGJpzH4KOBfYAKCqc4BWYQblnMte6dJQpYo9KlVKQAB798KQIVbEb9asAljL3MUqpktPqrpS9q+9khpOOM4VbVu22OdvdmJZJ1Tz51sJ8OnToUMHeOEFqFo1wUG5sMSSKFaKyCmAikgJoA+wKNywnCt63n0XunSJff2EthGvWGEDX4waBZ07exG/Qi6WRHE98AxQDVgFfA7cGGZQzhVFKSn2c9AgKFMm+/XjXil22jTrPNe7t/WHWLYMypePcxAuEWJJFMeo6uWRM0TkVOCbcEJyruho08au4gBs324/b7ghyerjbd9uNcyffhqOOsrGsC5VypNEERJLongOaBLDPOdcDk2aZB2XTzrJpmvVSrIk8dVXVrxv2TLLYI8/XoBHR3K5lWWiEJEWwCnA4SJyR8SiioDf9+ZcPmnXDh5+ONFRZCIlxa5v1a5tJTha+c2ORVW0M4qSQPlgncjvOH8AncIMyrnCbtQo+OAD66eWdGbNghNPtCJ+Y8dC69axNZq4QivLRKGqk4HJIjJcVX+NY0zOFXrPPw8zZsDxx8NppyU6msDvv1tv6nfftWtirVtbrz5X5MXSRrFDRAYDDYC/RhhR1TNDi8q5IqBFC/jyy0RHgRXxe+st6NMHtm2DgQML2JiqLmyxJIq3gHeA9titslcD68IMyrnCZvduePttGwUUrOxGrVoJDSndZZfZtbAWLayI33HHJToil2RiSRSVVfVVEekTcTnqh7ADc64w+eoruOaa/ecl9JJTZBG/c86xJHHTTV6fyWUqlkSxJ/j5m4icD6wGDg0vJOcKh9WrYfFiez5jhv384gsbMhqgcuXExMWSJXbL61VXWQG/Hj0SFIgrKGJJFANFpBJwJ9Z/oiJwW6hROVcIXHhheoJIU7MmHH54YuJh71548kl44AGrKOh3MrkYZZsoVPWT4OkW4Az4q2e2cy6KrVvhzDPh/vtt+uCDoW7dBAUzd65d+5oxAy6+2G67qlIlQcG4giZah7viQGesxtNnqjpfRNoD9wJlgBPjE6JzyUs1674QqjaYUOvW8Y0pUykpsHIlvPcedOzoRfxcjkQbj+JVoBdQGXhWRN4EhgD/VtWYkoSItBWRxSKyVET6ZbFOZxFZKCILROTtnL4B5xLplFPgoIMyf/z0U4Lbhr/9Fl56yZ6nFfHr1MmThMuxaJeemgGNVHWfiJQG1gBHq+qGWDYcnJE8D5wNpAA/iMgYVV0YsU5d4B7gVFXdJCKJHMzRuRz78UcbGvr88zNffvHF8Y0HsL4Q/fvDc8/B0UdbY3WpUlCuXAKCcYVBtETxp6ruA1DVXSKyLNYkETgJWKqqywBEZBTQAYgcCv5a4HlV3RTsZ22OoncuCTRvbsVVk8Lnn1sZ8BUr7HbXRx/1In4uz6IlimNFZG7wXICjg2kBVFUbZbPtasDKiOkUoHmGdeoBiMg3WKHBB1X1s4wbEpHeQG+AmjVrZrNb54qolSvt1Oboo2HKlCSqDeIKumiJIh7dMw8C6gKnA9WBKSLSUFU3R66kqkOBoQDNmjXTOMTlXFQffABvvGFXeRJuxgxo2hRq1IBx46BlS7v91bl8Eq0oYF4LAa4CakRMVw/mRUoBpqnqHmC5iCzBEof3/HZJ6aefrCvCp5/CunU2HOmZiap6tmYN3HILjB6dXsTv7LMTFIwrzKLd9ZRXPwB1RaS2iJQEugJjMqzzEXY2gYgchl2KWhZiTM7lyejRdiPRnj02js/s2dChQ5yDUIURI6B+fSsD/uijXsTPhSqWntm5oqp7ReRmYALW/vCaqi4QkQHAdFUdEyw7R0QWAqlA3xw2mDsXVxpc+Pz1VyhZMkFBdO1qpcBPPRVeeQWOPTZBgbiiIqZEISJlgJqqujgnG1fVccC4DPPuj3iuwB3BwzmXlcgifu3aWTvEjTdCsTAvCjhnsk0UInIB1tGuJFBbRBoDA1T1wrCDcy4ZLFxoHZvB2iji7scfoVcv6N7dfl59dQKCcEVZLGcUD2J9IiYBqOpsEakdYkzOJY3UVLuhaNeu9Hlly8bpi/yePTB4MDz0kHWWK18+Djt17kAxlRlX1S2yf7d/v0XVFQn79lmSuPZa+0IPULWqlegI1ezZ1qN69mwru/Hcc3DEESHv1LnMxfLnvkBELgOKByU3bgW+DTcs55LLkUfG+caiNWvs8f77cMklcdyxcweK5QT6Fmy87N3A21i5cR+Pwrn89vXX8MIL9rxtW/j5Z08SLinEkiiOVdX+qvqP4HGfqu7K/mXOFSyzZ8Pf/w4VK6Y/Dg3Gcgy1TWLrVrj5ZruT6emnbYBtsMYQ55JALJeenhCRI4DRwDuqOj/kmJyLq927oU0bu6Np7VobITRymNLixaFbt5B2PmGCFfFbuRL69IGBA72In0s6sYxwd0aQKDoDL4tIRSxhDAw9OufiYP16+OYbqwLbtSsMGQIlSsRhxytXQvv2UKeOXXby3tUuScV0Qq2qa1T1WeB6YDZwfzYvca7A6dkTnnkm5CShCt9/b89r1IDx42HWLE8SLqllmyhE5DgReVBE5gHPYXc8VQ89MucKm99+s2FImzeHyZNt3llneaVXl/RiaaN4DXgHOFdVV4ccj3OFjyoMHw533GGdMgYNsjpNzhUQsbRRtIhHIM6Fad8+ePVV2Lz5wGVbtoS8886drexsy5ZWxK9evZB36Fz+yjJRiMi7qto5uOQU2RM71hHunEuIjRttvIjU1PR5q1fbMNJZKVbMOtXlm9RUK+BXrBhccIENWnHddV7EzxVI0c4o+gQ/28cjEOfyy0svZZ0UvvzSmggyKl48H5sKFi2ylvEePaz2x1VX5dOGnUuMaCPc/RY8vVFV745cJiKDgLsPfJVziZfWX2358v3nlyljHepCs2ePtT88/LAV8KtUKcSdORc/sTRmn82BSeG8TOY5l1Rq1YrjzmbNsqqBc+dCly7w7LPwt7/FMQDnwhOtjeIG4EbgKBGZG7GoAvBN2IE5V6D8/rv13PvoowSMjepcuKKdUbwNjAceA/pFzN+qqhtDjcq5gmDKFJg3D266yYr4LV1q17ecK2SiJQpV1V9E5KaMC0TkUE8WLlns3QuNGqW3SezZE/LNRX/8Af36wYsv2q2uvXpZfSZPEq6Qyu6Moj0wA7s9NnLkIgWOCjEu52K2a5fdaHT66XDSSTbv2GND2tm4cXab6+rV1oFuwAAv4ucKvWh3PbUPfvqwpy7pfPghPPKIdXpO6y9x/vlw110h7nTlSmt/OOYY60CX2X22zhVCsdR6OlVEygXPrxCRJ0WkZvihOZe1zz+3G4yqVrXaehddZM0E+U4Vpk615zVq2I5nzvQk4YqUWG6PfRE4QUROAO4EXgHeAFqHGZhz2TnkEBg7NsQdrF4NN9wAY8bApEnQujWccUaIO3QuOcWSKPaqqopIB+D/VPVVEekZdmDORVKFxx+Hdets+pswb9BWtcJQd91lvfeGDPEifq5IiyVRbBWRe4ArgZYiUgyIx7AuzgHw2Wfw1VcweLC1G5csafNPPz2kHXbqBB98YGcQr7xiAws5V4TFkii6AJcB16jqmqB9YnC4YTmXrlcvWLUKDjrImghatQphJ5FF/C66CM45x+o0eRE/57JvzFbVNcBbQCURaQ/sUtXXQ4/MuUBqqtXY27kzpCQxf75dWnr1VZu+8kqv9OpchFjueuoMfA9cio2bPU1EOoUdmHORihe3M4p89eef8NBD0KQJ/PyztY475w4Qy79ef+AfqroWQEQOB74ARocZmHOhmjHDivjNnw+XXQZPPw2HH57oqJxLSrEkimJpSSKwgRjORJxLahs22HB3Y8dCex9yxbloYkkUn4nIBGBkMN0FGBdeSM6FZOJEK+J3663WWP3TT/k4WpFzhVcsjdl9gZeBRsFjaMaBjJxLalu2WOP0mWdaIb+0kY08STgXkywThYjUFZGPRWQ+1pD9hKreoaofxi88V5Q98QRUqWJDPYhkv36mxo6F+vWtP8Rdd1nbhBfxcy5Hop1RvAZ8AnTEKsg+F5eInAt8+619+e/d2/pS5NjKldCxI1SubPWaBg+GsmXzPU7nCrtobRQVVHVY8HyxiMyMR0DORapWDV56KQcvUIXvvoNTTkkv4nfKKcGP/6sAABuISURBVOnduZ1zORbtjKK0iJwoIk1EpAlQJsN0tkSkrYgsFpGlItIvynodRURFpFlO34Bzf0lJgQsvtM5zkyfbvNNP9yThXB5FO6P4DXgyYnpNxLQCZ0bbsIgUB54HzgZSgB9EZIyqLsywXgWgDzAtZ6E7F9i3D4YNg759bbi7J5+E005LdFTOFRrRBi7Kaz3lk4ClqroMQERGAR2AhRnWexgYBPTN4/5cUdWxI3z0kd3VNGwYHOWDLzqXn/K7KEKkasDKiOkUYL/RXoJLWDVU9VMRyTJRiEhvoDdAzZo+ZlJh9MUX1q0h0s8/R3nB3r1Wi6lYMUsU559vBaFyfXuUcy4rYSaKqIJy5U8C3bNbV1WHAkMBmjVrpuFG5sK0d6+1L+/Ysf/8yy+30ksZnXtuJhuZO9eSQq9e1j/iiitCidU5Z8JMFKuAGhHT1YN5aSoAxwOTxL4FHgGMEZELVXV6iHG5BPnxRxg/Hu64I/PlDz9slb0jHXpoxMTu3fDoo/Y45BCvzeRcnGSbKMQ+xS8HjlLVAcF4FEeo6vfZvPQHoK6I1MYSRFdsXAsAVHULcFjEfiYBd3mSKJy2bYOGDe2MAmDcOLt7NU3x4nDMMVEqe//wgxXxW7jQyoA/9ZT1j3DOhS6WM4oXgH3YXU4DgK3A+8A/or1IVfeKyM3ABKA48JqqLhCRAcB0VR2Tp8hdUtq27cDLSgAbN1qSuPlmuOYaOPHEHG540ybb+LhxcN55+RKrcy42sSSK5qraRERmAajqJhGJ6cZ0VR1HhgKCqnp/FuueHss2XfLQDK1Fa9fCkUeml1LKzPHH5yBJfPWVFfHr08eK+C1Z4uU3nEuAWBLFnqBPhMJf41HsCzUql/RGj4YuXawLQ0bXXguNGx84v0QJ6Nw5ho1v3mx9Il55BY47Dq6/3hKEJwnnEiKWRPEs8CHwNxF5BOgE3BdqVC5prVgBrVtbob59++Bf/7L2hTRlytjnesWKudzBxx/DDTfYDv75T3jwQU8QziVYtolCVd8SkRlAG0CAi1R1UeiRuaS0fDn88gt06ABnnGFXhfLNihVw6aV2FjFmDDTzii7OJYNY7nqqCewAxkbOU9UVYQbmklufPpYo8kwVvv4aWraEmjWt593JJ3t9JueSSCyXnj7F2icEKA3UBhYDDUKMyyWRESPsCz7AunX5uOEVK+w61fjxMGmSXdNq1Sofd+Ccyw+xXHpqGDkdlN24MbSIXMKtXw+PPAI7d9r0Bx/Y81q1bLpFCzj22DzsYN8+qx1+9912RvHss17Ez7kkluOe2ao6U0SaZ7+mKwjmzLE7mCItWgTvv2+9okuUsE5w/fpB//75tNNLLrFG67PPhqFD0zOQcy4pxdJGEVlwoRjQBFgdWkQuroYMgTffPLBH9GGHWcmNfOv8HFnEr0sXaw3v3t2L+DlXAMRyRlEh4vlerM3i/XDCcWH45Rcb9C0zy5ZBnToHVm7NV3PmWHfsa6+1Nolu3ULcmXMuv0VNFEFHuwqqelec4nEhuOUW+OSTrJeffHJIO961CwYOhEGD7DrWEUeEtCPnXJiyTBQiclBQr+nUeAbk8k9qqrU3rFtnZTNGjsx8vWrVQtj599/D1Vfb9aurr7ZR5/YrBeucKyiinVF8j7VHzBaRMcB7wPa0har6QcixuTx67jm4/XZ7ftZZVp01bv74w26V+uyzLAaVcM4VFLG0UZQGNmDVY9P6UyjgiSKJpababa5gt7c2aRKHnX7+OSxYYNnprLNg8WIvv+FcIRAtUfwtuONpPukJIo2PMpfEFi6Epk2tiaB4cbj44pB3uGmTjUY0fDg0aAA33uhF/JwrRKIliuJAefZPEGk8USSh1FQrj7RkiSWJ666zL/ah+uADuOkmawi55x64/35PEM4VMtESxW+qOiBukbiY7NhhH/5r1x64bN8+K9p32mlWh6l//5A/s1esgK5dbZCJceNyMRqRc64giJYovCdUElqzxvpEtGgBRx114PJWreDee6FevZACUIUpU6wuU82aNrhQ8+bWhds5VyhFSxRt4haFy9SCBTbeQ9o40wDbg/vOrr8erroqzgH9+qtdz5owIb2In9docq7QyzJRqOrGeAbi0u3caZeNvvnGuiM0arT/4EAnnxznoRr27YMXXrCCT2D33bZsGccAnHOJlOOigC58c+bAU09ZnaXTTrMv75GJIu4uugjGjrX+EC+/bANjO+eKDE8USWbsWPjvf+35m29C27YJCmTPHstOxYpZbaZOneDKK72In3NFkCeKJLFkCUydCr162Wd0iRJQtWqCgpk5E3r2tCJ+N97oRfycK+KKZb+Ki4cbb7SSSHv2wODBVgGjUaM4B7Fzp/WFOOkku72qRo04B+CcS0Z+RpEgqlawb8cOm16/3m55HTnS7jqN+xWeqVMtUy1ZYiXBhwyBQw6JcxDOuWTkiSKfbd+eXmMpmh9+gEsv3X9e+/YJbCfevt1OZ/773zh053bOFSSeKPJZ06ZWCy9WL74I1aunvzauPvvMOmvceSe0aWMlwUuWjHMQzrlk54kin/3+u33mXn559usefLDdeRr3y0wbNlgRv9dfh4YNbWSjkiU9STjnMuWJIgQNGkCPHomOIhOq8P77VsRv40a47z57eIJwzkXhdz3lkyFDoFYt2LIl0ZFEsWIFXHaZ3c00fTo8/LBXenXOZcsTRT6ZNAm2brUbh+JegykaVSvcB9ZSPmmS3eF0wgkJDcs5V3B4osijWbPgggtg2jSoXRv+858ENEpnZflyOOccazSZPNnmnXIKHORXHJ1zsfNEkUfjx8Mnn9iX9Yy3uyZMaio884yNEzFtmt1a5UX8nHO55F8tc+Hll+Gnn+z51Kn289tvk6hNuEMH+PRTaNcOXnrJe1g75/LEE0UOfPcdTJxoJcBLlEhPDCeemARXcyKL+F15pdVnuuwyL+LnnMuzUC89iUhbEVksIktFpF8my+8QkYUiMldEvhSRpKxf/fvv8Npr1kjdv7999o4YAdu22WPmTPt8Tpjp022AihdftOkuXawjhycJ51w+CO3jTUSKA88D5wH1gW4iUj/DarOAZqraCBgN/DusePLiySetmOrSpZYsdu5MkoKqO3fC3XfbUKTr1vk4Ec65UIR5weQkYKmqLgMQkVFAB2Bh2gqqOjFi/anAFSHGkyNLl9pnL8Avv0CFClbtomrVBA8ilOa77+xe3J9+strkgwdbV2/nnMtnYSaKasDKiOkUoHmU9XsC4zNbICK9gd4ANWvWzK/4MrV9uxVQbdbMRgBNU7VqkrUJ79xpAX7xhd3+6pxzIUl0EywAInIF0AxondlyVR0KDAVo1qyZ5vf+t22zO0oBzjzT2hzAruqccYY9P+qo/N5rLowbZ6c1fftaoIsWWau6c86FKMxEsQqI/A5ePZi3HxE5C+gPtFbV3SHGk6kxY+xu0khnnAHXX28d6cqUiXdEmVi/Hm67Dd56y3pU9+ljt1x5knDOxUGYieIHoK6I1MYSRFfgssgVRORE4GWgraquDTGWLK0MLo4NGADly9vzCy+Eo49ORDQZqMI771h11y1b4IEH4N57k6jDhnOuKAgtUajqXhG5GZgAFAdeU9UFIjIAmK6qY4DBQHngPbFbOVeo6oVhxRTN9dfD4YcnYs9RrFhhDdYnnACvvmolwZ1zLs5CbaNQ1XHAuAzz7o947kOpZaQKX35po8wdeaTVaPrHP5LkVivnXFHktZ6Syc8/2x1MZ5+dXsTv5JM9STjnEsoTRTJITbVefQ0bwowZVkzKi/g555JEUtwemwh799q4PWlf3BPqggusDG379vsPou2cc0mgyCaKBQvsTqeyZa0ad6VKcQ7gzz+tkmCxYtC9uxXy69rV6zM555JOkb30pEG3vbfegnnz4nzH6fff2+hGL7xg0507W/EoTxLOuSRUZBNFQuzYAXfeCS1awKZNSdJZwznnoiuyl57i7uuvrU/EsmVw3XUwaFACrnc551zOFalEsWuXDSMN9nkdV2kDC02cCKefHuedO+dc7hX6RLF1K+wOKkhddx188MH+y0Ot5TR2rBXu++c/rYDUwoVJMBSec87lTKH+1Jo7F5o0Sa8MC9CgAfzrX/a8bNmQKnSvW2eF+0aOhMaNraBfyZKeJJxzBVKh/uRas8aSxO23p5cJb9nSSieFQtWSw623wh9/2P23d9/tRfyccwVaoU4UaTp1glNOicOOVqyAHj3gxBOtiF+DBnHYqXPOhctvj82rfftgwgR7fuSR8L//wTffeJJwzhUanijy4qefbKS5tm1hyhSbd9JJXsTPOVeoeKLIjb17YfBgaNQIZs+2y0xexM85V0gViTaKfNe+vV1u6tDBynBUrZroiJxLSnv27CElJYVdu3YlOpQio3Tp0lSvXp0S+ThUsieKWO3ebWNUFysGvXrBNdfApZd6fSbnokhJSaFChQrUqlUL8f+V0KkqGzZsICUlhdq1a+fbdv3SUyymTrUOGc8/b9OdOlkhP//Ddy6qXbt2UblyZU8ScSIiVK5cOd/P4Ar8GcXu3fDZZ1aeI6M5c/K48e3b4b774JlnbIyIunXzuEHnih5PEvEVxvEu8Ili7Fi7AhTNoYfmYsP/+58V8Vu+HG68ER57DCpWzFWMzjlXkBX4S09pZxLjx1sppYyPlSvh2GNzseG9e61NYvJku+TkScK5Auujjz5CRPjxxx//mjdp0iTat2+/33rdu3dn9OjRgDXE9+vXj7p169KkSRNatGjB+PHj8xzLY489Rp06dTjmmGOYkNYHK4Pu3btTu3ZtGjduTOPGjZk9ezYAgwcP/mve8ccfT/Hixdm4cWOeY8pOgT6j2LYNNm+253Xq2CNPPvrIivjdc48V8VuwwOszOVcIjBw5ktNOO42RI0fy0EMPxfSaf/3rX/z222/Mnz+fUqVK8fvvvzM5j2MnL1y4kFGjRrFgwQJWr17NWWedxZIlSyieSd+rwYMH06lTp/3m9e3bl759+wIwduxYnnrqKQ7N1SWTnCmwn4IbN0K1aulnFHkqp/T773DLLfDee9ZofeedXsTPuXx2223W7Sg/NW4MTz8dfZ1t27bx9ddfM3HiRC644IKYEsWOHTsYNmwYy5cvp1SpUgD8/e9/p3PnznmK9+OPP6Zr166UKlWK2rVrU6dOHb7//ntatGiR422NHDmSbt265SmeWBXYS0+bN1uSuPpqGD0aatbMxUZU4Y03oH59+PhjeOQRu8PJi/g5V2h8/PHHtG3blnr16lG5cmVmzJiR7WuWLl1KzZo1qRjDJefbb7/9r8tBkY/HH3/8gHVXrVpFjRo1/pquXr06q1atynS7/fv3p1GjRtx+++3sThsrIbBjxw4+++wzOnbsmG18+aHAf2U+80zI9bFascL6RDRrZr2rc9WY4ZyLRXbf/MMycuRI+vTpA0DXrl0ZOXIkTZs2zfLuoJzeNfTUU0/lOcaMHnvsMY444gj+/PNPevfuzaBBg7j//vv/Wj527FhOPfXUuFx2gkKQKHIsrYjfeedZEb9vvrFqr16fyblCZ+PGjXz11VfMmzcPESE1NRURYfDgwVSuXJlNmzYdsP5hhx1GnTp1WLFiBX/88Ue2ZxW33347EydOPGB+165d6dev337zqlWrxsqVK/+aTklJoVq1age8tkqVKgCUKlWKHj16MGTIkP2Wjxo1Km6XnQDryVeQHk2bNlVV1Z9/VgXVESM0dosXq7ZsaS+cNCkHL3TO5cbChQsTuv+XX35Ze/fuvd+8Vq1a6eTJk3XXrl1aq1atv2L85ZdftGbNmrp582ZVVe3bt692795dd+/eraqqa9eu1XfffTdP8cyfP18bNWqku3bt0mXLlmnt2rV17969B6y3evVqVVXdt2+f9unTR+++++6/lm3evFkPOeQQ3bZtW5b7yey4A9M1l5+7BbaNIkf27oVBg6yI37x58J//QKtWiY7KOReykSNHcvHFF+83r2PHjowcOZJSpUrx5ptv0qNHDxo3bkynTp145ZVXqFSpEgADBw7k8MMPp379+hx//PG0b98+pjaLaBo0aEDnzp2pX78+bdu25fnnn//rjqd27dqxevVqAC6//HIaNmxIw4YNWb9+Pffdd99f2/jwww8555xzKFeuXJ5iyQmxRFNwNGvWTG+4YTqffw7vvgsjRsBVV2XzonPPhc8/h0susT4RRxwRl1idK+oWLVrEcccdl+gwipzMjruIzFDVZrnZXoFro1i1yoaj3rcPateG44/PYsVdu6zDXPHi0Lu3PeJ0h4BzzhUmBe7S05o1sGePXUlatsy6PRzgm2/sBuu0In4dO3qScM65XCpwZxSlS8POnVks3LYN7r0X/u//rGOFn/I6l3Cq6oUB4yiM5oQCd0aRpcmT7TrU//0f3HwzzJ8PZ5+d6KicK9JKly7Nhg0bQvnwcgfSYDyK0qVL5+t2C9wZRVRly1rV11NPTXQkzjms53FKSgrr1q1LdChFRtoId/mpwN31VKZMM925c7pNfPAB/PijXW4CSE31jnPOOZeJvNz1FOqlJxFpKyKLRWSpiPTLZHkpEXknWD5NRGrFtOE1a2yUuY4d4cMP4c8/bb4nCeecy3ehJQoRKQ48D5wH1Ae6iUj9DKv1BDapah3gKWBQdts9OHWDNVJ/8okNJvTtt17EzznnQhTmGcVJwFJVXaaqfwKjgA4Z1ukAjAiejwbaSDa3R1Td86s1Ws+ZA/36WV8J55xzoQmzMbsasDJiOgVontU6qrpXRLYAlYH1kSuJSG+gdzC5W77+er5XegXgMDIcqyLMj0U6Pxbp/FikOya3LywQdz2p6lBgKICITM9tg0xh48cinR+LdH4s0vmxSCci03P72jAvPa0CakRMVw/mZbqOiBwEVAI2hBiTc865HAozUfwA1BWR2iJSEugKjMmwzhjg6uB5J+ArLWj36zrnXCEX2qWnoM3hZmACUBx4TVUXiMgArC76GOBV4A0RWQpsxJJJdoaGFXMB5McinR+LdH4s0vmxSJfrY1HgOtw555yLr8JT68k551woPFE455yLKmkTRWjlPwqgGI7FHSKyUETmisiXInJkIuKMh+yORcR6HUVERaTQ3hoZy7EQkc7B38YCEXk73jHGSwz/IzVFZKKIzAr+T9olIs6wichrIrJWROZnsVxE5NngOM0VkcxG9DlQbgfbDvOBNX7/DBwFlATmAPUzrHMj8FLwvCvwTqLjTuCxOAMoGzy/oSgfi2C9CsAUYCrQLNFxJ/Dvoi4wCzgkmP5bouNO4LEYCtwQPK8P/JLouEM6Fq2AJsD8LJa3A8YDApwMTItlu8l6RhFK+Y8CKttjoaoTVXVHMDkV67NSGMXydwHwMFY3bFc8g4uzWI7FtcDzqroJQFXXxjnGeInlWChQMXheCVgdx/jiRlWnYHeQZqUD8LqaqcDBIlIlu+0ma6LIrPxHtazWUdW9QFr5j8ImlmMRqSf2jaEwyvZYBKfSNVT103gGlgCx/F3UA+qJyDciMlVE2sYtuviK5Vg8CFwhIinAOOCW+ISWdHL6eQIUkBIeLjYicgXQDGid6FgSQUSKAU8C3RMcSrI4CLv8dDp2ljlFRBqq6uaERpUY3YDhqvqEiLTA+m8dr6r7Eh1YQZCsZxRe/iNdLMcCETkL6A9cqKq74xRbvGV3LCoAxwOTROQX7BrsmELaoB3L30UKMEZV96jqcmAJljgKm1iORU/gXQBV/Q4ojRUMLGpi+jzJKFkThZf/SJftsRCRE4GXsSRRWK9DQzbHQlW3qOphqlpLVWth7TUXqmqui6ElsVj+Rz7CziYQkcOwS1HL4hlknMRyLFYAbQBE5DgsURTF8VnHAFcFdz+dDGxR1d+ye1FSXnrS8Mp/FDgxHovBQHngvaA9f4WqXpiwoEMS47EoEmI8FhOAc0RkIZAK9FXVQnfWHeOxuBMYJiK3Yw3b3QvjF0sRGYl9OTgsaI95ACgBoKovYe0z7YClwA6gR0zbLYTHyjnnXD5K1ktPzjnnkoQnCuecc1F5onDOOReVJwrnnHNReaJwzjkXlScKl5REJFVEZkc8akVZd1s+7G+4iCwP9jUz6L2b0228IiL1g+f3Zlj2bV5jDLaTdlzmi8hYETk4m/UbF9ZKqS5+/PZYl5REZJuqls/vdaNsYzjwiaqOFpFzgCGq2igP28tzTNltV0RGAEtU9ZEo63fHKujenN+xuKLDzyhcgSAi5YOxNmaKyDwROaBqrIhUEZEpEd+4WwbzzxGR74LXvici2X2ATwHqBK+9I9jWfBG5LZhXTkQ+FZE5wfwuwfxJItJMRB4HygRxvBUs2xb8HCUi50fEPFxEOolIcREZLCI/BOMEXBfDYfmOoKCbiJwUvMdZIvKtiBwT9FIeAHQJYukSxP6aiHwfrJtZ9V3n9pfo+un+8EdmD6wn8ezg8SFWRaBisOwwrGdp2hnxtuDnnUD/4HlxrPbTYdgHf7lg/t3A/ZnsbzjQKXh+KTANaArMA8phPd8XACcCHYFhEa+tFPycRDD+RVpMEeukxXgxMCJ4XhKr5FkG6A3cF8wvBUwHamcS57aI9/ce0DaYrggcFDw/C3g/eN4d+L+I1z8KXBE8Pxir/1Qu0b9vfyT3IylLeDgH7FTVxmkTIlICeFREWgH7sG/SfwfWRLzmB+C1YN2PVHW2iLTGBqr5JihvUhL7Jp6ZwSJyH1YDqCdWG+hDVd0exPAB0BL4DHhCRAZhl6v+l4P3NR54RkRKAW2BKaq6M7jc1UhEOgXrVcIK+C3P8PoyIjI7eP+LgP9GrD9CROpiJSpKZLH/c4ALReSuYLo0UDPYlnOZ8kThCorLgcOBpqq6R6w6bOnIFVR1SpBIzgeGi8iTwCbgv6raLYZ99FXV0WkTItIms5VUdYnYuBftgIEi8qWqDojlTajqLhGZBJwLdMEG2QEbcewWVZ2QzSZ2qmpjESmL1Ta6CXgWG6xpoqpeHDT8T8ri9QJ0VNXFscTrHHgbhSs4KgFrgyRxBnDAuOBiY4X/rqrDgFewISGnAqeKSFqbQzkRqRfjPv8HXCQiZUWkHHbZ6H8iUhXYoapvYgUZMxt3eE9wZpOZd7BibGlnJ2Af+jekvUZE6gX7zJTaiIa3AndKepn9tHLR3SNW3YpdgkszAbhFgtMrscrDzkXlicIVFG8BzURkHnAV8GMm65wOzBGRWdi39WdUdR32wTlSROZil52OjWWHqjoTa7v4HmuzeEVVZwENge+DS0APAAMzeflQYG5aY3YGn2ODS32hNnQnWGJbCMwUkflY2fioZ/xBLHOxQXn+DTwWvPfI100E6qc1ZmNnHiWC2BYE085F5bfHOueci8rPKJxzzkXlicI551xUniicc85F5YnCOedcVJ4onHPOReWJwjnnXFSeKJxzzkX1/wFdxooQBECRAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Compute predicted probabilities\n",
        "\n",
        "# X_data = pd.concat([X_train_tfidf, X_val_tfidf], axis=0)\n",
        "# y_data = pd.concat([X_train_tfidf, X_val_tfidf], axis=0)\n",
        "\n",
        "X_test = test_data.text.values\n",
        "y_test = test_data.isRumor.values\n",
        "\n",
        "X_preprocessed = np.array([text_preprocessing(text) for text in X])\n",
        "X_tfidf = tf_idf.transform(X_preprocessed)\n",
        "X_test_preprocessed = np.array([text_preprocessing(text) for text in X_test])\n",
        "X_test_tfidf = tf_idf.transform(X_test_preprocessed)\n",
        "\n",
        "nb_model = MultinomialNB(alpha=1.8)\n",
        "nb_model.fit(X_tfidf, y)\n",
        "probs = nb_model.predict_proba(X_test_tfidf)\n",
        "\n",
        "# Evaluate the classifier\n",
        "evaluate_roc(probs, test_data.isRumor)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "BCYueNHxlO5C",
        "outputId": "a05dc3c3-72e8-4665-afc6-85b7249591d9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.49      1.00      0.66       189\n",
            "           1       1.00      0.03      0.06       201\n",
            "\n",
            "    accuracy                           0.50       390\n",
            "   macro avg       0.75      0.51      0.36       390\n",
            "weighted avg       0.75      0.50      0.35       390\n",
            "\n"
          ]
        }
      ],
      "source": [
        "from sklearn.metrics import classification_report\n",
        "preds = np.argmax(probs, axis = 1)\n",
        "print(classification_report(test_data.isRumor, preds))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "-GBWZFMnrpO6"
      },
      "source": [
        "# 공통 Functions"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "LYT-KkTor3QE"
      },
      "source": [
        "## Library"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 34,
      "metadata": {
        "id": "VCz_QkG8ruDj"
      },
      "outputs": [],
      "source": [
        "import re\n",
        "from tqdm import tqdm\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import matplotlib.pyplot as plt\n",
        "import time\n",
        "\n",
        "%matplotlib inline\n",
        "from transformers import BertTokenizer\n",
        "from sklearn.metrics import accuracy_score, roc_curve, auc, classification_report, f1_score\n",
        "\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "import torch.nn.functional as F\n",
        "\n",
        "import time\n",
        "\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "from transformers import BertModel\n",
        "from transformers import AdamW, get_linear_schedule_with_warmup\n",
        "\n",
        "import random\n",
        "import time\n",
        "\n",
        "from transformers import BertTokenizer\n",
        "from sklearn.metrics import accuracy_score, roc_curve, auc, precision_score, recall_score, f1_score\n",
        "from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler\n",
        "\n",
        "from transformers import AutoModel, AutoTokenizer\n",
        "\n",
        "from fetchData import fetchdata \n",
        "\n",
        "import emoji"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RkOFN0Tpp5s"
      },
      "source": [
        "## Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 35,
      "metadata": {
        "id": "UgmOuPxRFhof"
      },
      "outputs": [],
      "source": [
        "def getDevice():\n",
        "  if torch.cuda.is_available():       \n",
        "      device = torch.device(\"cuda\")\n",
        "      print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "      print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "  else:\n",
        "      print('No GPU available, using the CPU instead.')\n",
        "      device = torch.device(\"cpu\")\n",
        "  return device\n",
        "\n",
        "def evaluate_roc(probs, y_true):\n",
        "    \"\"\"\n",
        "    - Print AUC and accuracy on the test set\n",
        "    - Plot ROC\n",
        "    @params    probs (np.array): an array of predicted probabilities with shape (len(y_true), 2)\n",
        "    @params    y_true (np.array): an array of the true values with shape (len(y_true),)\n",
        "    \"\"\"\n",
        "    preds = probs[:, 1]\n",
        "    fpr, tpr, threshold = roc_curve(y_true, preds)\n",
        "    roc_auc = auc(fpr, tpr)\n",
        "    print(f'AUC: {roc_auc:.4f}')\n",
        "\n",
        "    # Get accuracy over the test set\n",
        "    y_pred = np.where(preds >= 0.5, 1, 0)\n",
        "    accuracy = accuracy_score(y_true, y_pred)\n",
        "    print(f'Accuracy: {accuracy*100:.2f}%')\n",
        "\n",
        "    # Plot ROC AUC\n",
        "    plt.title('Receiver Operating Characteristic')\n",
        "    plt.plot(fpr, tpr, 'b', label='AUC = %0.2f' % roc_auc)\n",
        "    plt.legend(loc='lower right')\n",
        "    plt.plot([0, 1], [0, 1], 'r--')\n",
        "    plt.xlim([0, 1])\n",
        "    plt.ylim([0, 1])\n",
        "    plt.ylabel('True Positive Rate')\n",
        "    plt.xlabel('False Positive Rate')\n",
        "    plt.show()\n",
        "    \n",
        "def set_seed(seed_value=42):\n",
        "    \"\"\"Set seed for reproducibility.\n",
        "    \"\"\"\n",
        "    random.seed(seed_value)\n",
        "    np.random.seed(seed_value)\n",
        "    torch.manual_seed(seed_value)\n",
        "    torch.cuda.manual_seed_all(seed_value)\n",
        "\n",
        "def preprocessing_for_bert(data): \n",
        "\n",
        "    \"\"\"Perform required preprocessing steps for pretrained BERT.\n",
        "    @param    data (np.array): Array of texts to be processed.\n",
        "    @return   input_ids (torch.Tensor): Tensor of token ids to be fed to a model.\n",
        "    @return   attention_masks (torch.Tensor): Tensor of indices specifying which\n",
        "                  tokens should be attended to by the model.\n",
        "    \"\"\"\n",
        "    # Create empty lists to store outputs (빈 리스트 2개 생성)\n",
        "    input_ids = []\n",
        "    attention_masks = []\n",
        "\n",
        "    # For every sentence...\n",
        "    for sent in data:\n",
        "        # `encode_plus` will:\n",
        "        #    (1) Tokenize the sentence\n",
        "        #    (2) Add the `[CLS]` and `[SEP]` token to the start and end\n",
        "        #    (3) Truncate/Pad sentence to max length\n",
        "        #    (4) Map tokens to their IDs\n",
        "        #    (5) Create attention mask\n",
        "        #    (6) Return a dictionary of outputs\n",
        "        encoded_sent = tokenizer.encode_plus(\n",
        "            text=text_preprocessing(sent),  # Preprocess sentence\n",
        "            add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "            # max_length=MAX_LEN,                  # Max length to truncate/pad\n",
        "            pad_to_max_length=True,         # Pad sentence to max length\n",
        "            # return_tensors='pt',           # Return PyTorch tensor\n",
        "            return_attention_mask=True,      # Return attention mask\n",
        "\n",
        "            # max_length=True,                  # Max length to truncate/pad\n",
        "            padding='max_length'\n",
        "        )\n",
        "\n",
        "        # Add the outputs to the lists (위의 빈 리스트에 상응하는 값 추가)\n",
        "        input_ids.append(encoded_sent.get('input_ids'))\n",
        "        attention_masks.append(encoded_sent.get('attention_mask'))\n",
        "\n",
        "    # Convert lists to tensors (리스트들을 텐서화)\n",
        "    input_ids = torch.tensor(input_ids)\n",
        "    attention_masks = torch.tensor(attention_masks)\n",
        "\n",
        "    return input_ids, attention_masks\n",
        "\n",
        "def train(model, train_dataloader, val_dataloader=None, epochs=4, evaluation=False):\n",
        "    \"\"\"Train the BertClassifier model.\n",
        "    \"\"\"\n",
        "    best_acc = 0\n",
        "    # Start training loop\n",
        "    print(\"Start training...\\n\")\n",
        "    for epoch_i in range(epochs):\n",
        "        # =======================================\n",
        "        #               Training\n",
        "        # =======================================\n",
        "        # Print the header of the result table\n",
        "        print(f\"{'Epoch':^7} | {'Batch':^7} | {'Train Loss':^12} | {'Val Loss':^10} | {'Val Acc':^9} | {'Elapsed':^9}\")\n",
        "        print(\"-\"*70)\n",
        "\n",
        "        # Measure the elapsed time of each epoch\n",
        "        t0_epoch, t0_batch = time.time(), time.time()\n",
        "\n",
        "        # Reset tracking variables at the beginning of each epoch\n",
        "        total_loss, batch_loss, batch_counts = 0, 0, 0\n",
        "\n",
        "        # Put the model into the training mode\n",
        "        model.train()\n",
        "\n",
        "        # For each batch of training data...\n",
        "        for step, batch in enumerate(train_dataloader):\n",
        "            batch_counts += 1\n",
        "            # Load batch to GPU\n",
        "            b_input_ids, b_attn_mask, b_labels = tuple(\n",
        "                t.to(device) for t in batch)\n",
        "\n",
        "            # Zero out any previously calculated gradients\n",
        "            model.zero_grad()\n",
        "\n",
        "            # Perform a forward pass. This will return logits.\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "            # Compute loss and accumulate the loss values\n",
        "            loss = loss_fn(logits, b_labels)\n",
        "            batch_loss += loss.item()\n",
        "            total_loss += loss.item()\n",
        "\n",
        "            # Perform a backward pass to calculate gradients\n",
        "            loss.backward()\n",
        "\n",
        "            # Clip the norm of the gradients to 1.0 to prevent \"exploding gradients\"\n",
        "            torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)\n",
        "\n",
        "            # Update parameters and the learning rate\n",
        "            optimizer.step()\n",
        "            scheduler.step()\n",
        "\n",
        "            # Print the loss values and time elapsed for every 20 batches\n",
        "            if (step % 20 == 0 and step != 0) or (step == len(train_dataloader) - 1):\n",
        "                # Calculate time elapsed for 20 batches\n",
        "                time_elapsed = time.time() - t0_batch\n",
        "\n",
        "                # Print training results\n",
        "                print(\n",
        "                    f\"{epoch_i + 1:^7} | {step:^7} | {batch_loss / batch_counts:^12.6f} | {'-':^10} | {'-':^9} | {time_elapsed:^9.2f}\")\n",
        "\n",
        "                # Reset batch tracking variables\n",
        "                batch_loss, batch_counts = 0, 0\n",
        "                t0_batch = time.time()\n",
        "\n",
        "        # Calculate the average loss over the entire training data\n",
        "        avg_train_loss = total_loss / len(train_dataloader)\n",
        "\n",
        "        print(\"-\"*70)\n",
        "        # =======================================\n",
        "        #               Evaluation\n",
        "        # =======================================\n",
        "        if evaluation == True:\n",
        "            # After the completion of each training epoch, measure the model's performance\n",
        "            # on our validation set.\n",
        "            val_loss, val_accuracy = evaluate(model, val_dataloader)\n",
        "\n",
        "            # Print performance over the entire training data\n",
        "            time_elapsed = time.time() - t0_epoch\n",
        "\n",
        "            print(f\"{epoch_i + 1:^7} | {'-':^7} | {avg_train_loss:^12.6f} | {val_loss:^10.6f} | {val_accuracy:^9.2f} | {time_elapsed:^9.2f}\")\n",
        "            print(\"-\"*70)\n",
        "        print(\"\\n\")\n",
        "\n",
        "        # early_stopping needs the validation loss to check if it has decresed, \n",
        "        # and if it has, it will make a checkpoint of the current model\n",
        "        \n",
        "        if val_accuracy > best_acc:\n",
        "            best_acc = val_accuracy\n",
        "            es = 0\n",
        "            # torch.save(model.state_dict(), \"model_\" + str(epoch_i) + 'weight.pt')\n",
        "            torch.save(model.state_dict(), \"checkpoint.pt\")\n",
        "        else:\n",
        "            es += 1\n",
        "            print(\"Counter {} of 5\".format(es))\n",
        "\n",
        "            if es > 1:\n",
        "                print(\"Early stopping with best_acc: \", best_acc, \"and val_acc for this epoch: \", val_accuracy, \"...\")\n",
        "                break\n",
        "\n",
        "    model.load_state_dict(torch.load(\"checkpoint.pt\"))\n",
        "    print(\"Training complete!\")\n",
        "\n",
        "def evaluate(model, val_dataloader):\n",
        "    \"\"\"After the completion of each training epoch, measure the model's performance\n",
        "    on our validation set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    # Tracking variables\n",
        "    val_accuracy = []\n",
        "    val_loss = []\n",
        "\n",
        "    # For each batch in our validation set...\n",
        "    for batch in val_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask, b_labels = tuple(t.to(device) for t in batch)\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "\n",
        "        # Compute loss\n",
        "        loss = loss_fn(logits, b_labels)\n",
        "        val_loss.append(loss.item())\n",
        "\n",
        "        # Get the predictions\n",
        "        preds = torch.argmax(logits, dim=1).flatten()\n",
        "\n",
        "        # Calculate the accuracy rate\n",
        "        accuracy = (preds == b_labels).cpu().numpy().mean() * 100\n",
        "        val_accuracy.append(accuracy)\n",
        "\n",
        "    # Compute the average accuracy and loss over the validation set.\n",
        "    val_loss = np.mean(val_loss)\n",
        "    val_accuracy = np.mean(val_accuracy)\n",
        "\n",
        "    return val_loss, val_accuracy\n",
        "\n",
        "def bert_predict(model, test_dataloader):\n",
        "    \"\"\"Perform a forward pass on the trained BERT model to predict probabilities\n",
        "    on the test set.\n",
        "    \"\"\"\n",
        "    # Put the model into the evaluation mode. The dropout layers are disabled during\n",
        "    # the test time.\n",
        "    model.eval()\n",
        "\n",
        "    all_logits = []\n",
        "\n",
        "    # For each batch in our test set...\n",
        "    for batch in test_dataloader:\n",
        "        # Load batch to GPU\n",
        "        b_input_ids, b_attn_mask = tuple(t.to(device) for t in batch)[:2]\n",
        "\n",
        "        # Compute logits\n",
        "        with torch.no_grad():\n",
        "            logits = model(b_input_ids, b_attn_mask)\n",
        "        all_logits.append(logits)\n",
        "    \n",
        "    # Concatenate logits from each batch\n",
        "    all_logits = torch.cat(all_logits, dim=0)\n",
        "\n",
        "    # Apply softmax to calculate probabilities\n",
        "    probs = F.softmax(all_logits, dim=1).cpu().numpy()\n",
        "\n",
        "    return probs\n",
        "\n",
        "def data_process(X_train, y_train, X_val, y_val, batch_size=32):\n",
        "  # Concatenate train data and test data\n",
        "  all_tweets = np.concatenate([X_train, X_val])\n",
        "\n",
        "  # Encode our concatenated data\n",
        "  encoded_tweets = [tokenizer.encode(sent, add_special_tokens=True) for sent in all_tweets]\n",
        "\n",
        "  # Find the maximum length\n",
        "  max_len = max([len(sent) for sent in encoded_tweets])\n",
        "  print('Max length: ', max_len)\n",
        "\n",
        "  # Specify `MAX_LEN`\n",
        "  MAX_LEN = max_len\n",
        "\n",
        "  # Run function `preprocessing_for_bert` on the train set and the validation set\n",
        "  print('\\nTokenizing data...')\n",
        "  train_inputs, train_masks = preprocessing_for_bert(X_train)\n",
        "  val_inputs, val_masks = preprocessing_for_bert(X_val)\n",
        "\n",
        "  # Convert other data types to torch.Tensor\n",
        "  train_labels = torch.tensor(y_train)\n",
        "  val_labels = torch.tensor(y_val)\n",
        "\n",
        "  # For fine-tuning BERT, the authors recommend a batch size of 16 or 32.\n",
        "  batch_size = 8\n",
        "\n",
        "  # Create the DataLoader for our training set\n",
        "  train_data = TensorDataset(train_inputs, train_masks, train_labels)\n",
        "  train_sampler = RandomSampler(train_data)\n",
        "  train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=batch_size)\n",
        "\n",
        "  # Create the DataLoader for our validation set\n",
        "  val_data = TensorDataset(val_inputs, val_masks, val_labels)\n",
        "  val_sampler = SequentialSampler(val_data)\n",
        "  val_dataloader = DataLoader(val_data, sampler=val_sampler, batch_size=batch_size)\n",
        "\n",
        "  return train_dataloader, val_dataloader\n",
        "\n",
        "def train_process(train_dataloader, val_dataloader, epoch=4):\n",
        "  set_seed(42)    # Set seed for reproducibility\n",
        "  bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=epoch)\n",
        "\n",
        "  train(bert_classifier, train_dataloader, loss_fn, epochs=4, evaluation=True)\n",
        "\n",
        "  return bert_classifier\n",
        "\n",
        "def testing_process(bert_classifier, X_val, y_val):\n",
        "  #  Run `preprocessing_for_bert` on the test set\n",
        "  print('Tokenizing data...')\n",
        "  test_inputs, test_masks = preprocessing_for_bert(X_val)\n",
        "\n",
        "  # Create the DataLoader for our test set\n",
        "  test_dataset = TensorDataset(test_inputs, test_masks)\n",
        "  test_sampler = SequentialSampler(test_dataset)\n",
        "  test_dataloader = DataLoader(test_dataset, sampler=test_sampler, batch_size=32)\n",
        "\n",
        "  # Compute predicted probabilities on the test set\n",
        "  probs = bert_predict(bert_classifier, test_dataloader)\n",
        "\n",
        "  # Get predictions from the probabilities\n",
        "  threshold = 0.5\n",
        "  preds = np.where(probs[:, 1] > threshold, 1, 0)\n",
        "\n",
        "  # Number of tweets predicted non-negative\n",
        "  print(\"Number of tweets predicted as Rumor: \", preds.sum())\n",
        "\n",
        "  preds = np.argmax(probs, axis = 1)\n",
        "  print(\"\\n\",classification_report(y_val, preds))\n",
        "  print('Accuracy Score:\\t',accuracy_score(y_val, preds))\n",
        "  print('Precision Score:\\t', str(precision_score(y_val,preds)))\n",
        "  print('Recall Score:\\t\\t' + str(recall_score(y_val,preds)))\n",
        "  print('F1 Score:\\t',f1_score(y_val, preds, zero_division=1))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zNAzuZdm-5eY"
      },
      "source": [
        "# BERT 최종\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4Pr-Y1EFtqwy"
      },
      "source": [
        "## Class and Functions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 38,
      "metadata": {
        "id": "81xpX8IxtqMZ"
      },
      "outputs": [],
      "source": [
        "def text_preprocessing(text): # Create a function to tokenize a set of texts\n",
        "\n",
        "    \"\"\"\n",
        "    - Remove entity mentions (eg. '@united')\n",
        "    - Correct errors (eg. '&amp;' to '&')\n",
        "    @param    text (str): a string to be processed.\n",
        "    @return   text (Str): the processed string.\n",
        "    \"\"\"\n",
        "    # Remove '@name'\n",
        "    text = text.lower()\n",
        "    # text = re.sub(r'(@.*?)[\\s]', ' ', text)\n",
        "    text = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', text) # 해쉬태그 띄우기\n",
        "\n",
        "    # text = re.sub(r\"http\\S+\", \"*\", text)  # http link -> '*'\n",
        "\n",
        "    # text = re.sub(r\"@\\S+\", \"@\", text)   # mention -> '@'\n",
        "    # text = re.sub(r\"@[^\\s]+\", \"@\", text)   # mention -> '@'\n",
        "\n",
        "    # sent = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', sent)\n",
        "    # sent = re.sub(r'([^\\s\\w@#\\*]|_)+', '', sent) # Erasing Special Characters\n",
        "\n",
        "    # Replace '&amp;' with '&'\n",
        "    text = re.sub(r'&amp;', '&', text)\n",
        "\n",
        "    # Remove trailing whitespace\n",
        "    text = re.sub(r'\\s+', ' ', text).strip()\n",
        "\n",
        "    return text\n",
        "    \n",
        "def initialize_model(epochs=4):\n",
        "    \"\"\"Initialize the Bert Classifier, the optimizer and the learning rate scheduler.\n",
        "    \"\"\"\n",
        "    # Instantiate Bert Classifier\n",
        "    bert_classifier = BertClassifier(freeze_bert=True)\n",
        "\n",
        "    # Tell PyTorch to run the model on GPU\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(bert_classifier.parameters(),\n",
        "                      # lr=5e-5,    # Default learning rate\n",
        "                      lr=2e-5,    # Default learning rate\n",
        "                      eps=1e-8    # Default epsilon value\n",
        "                      )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0,  # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler, criterion\n",
        "\n",
        "class BertClassifier(nn.Module): # Create the BertClassfier class\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 50, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = BertModel.from_pretrained('bert-base-uncased')\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            # nn.Dropout(0.5),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "\n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "fk3m9D6kpsOP"
      },
      "source": [
        "## Executions"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 39,
      "metadata": {
        "id": "VAAH77uL_FiS"
      },
      "outputs": [],
      "source": [
        "raw_text = pd.read_csv('./data/_PHEME_text.csv')\n",
        "y = pd.read_csv('./data/_PHEME_target.csv')\n",
        "data = pd.concat([raw_text.text, y], axis=1).reset_index(drop=True)\n",
        "val = pd.read_csv('data/_PHEMEext_text.csv')\n",
        "\n",
        "X_train = data.text.values\n",
        "y_train = data.target.values\n",
        "\n",
        "X_val = val.drop(['Event'],axis=1).text.values\n",
        "y_val = val.target.values\n",
        "\n",
        "rhi_data = pd.read_csv('data/_RHI_text.csv').text.values\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv').isRumor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "K84HzvLMbP0X",
        "outputId": "2fc6e130-0823-473d-9da0-a03b2bf57adf"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Max length:  69\n",
            "\n",
            "Tokenizing data...\n"
          ]
        }
      ],
      "source": [
        "# device = getDevice()\n",
        "# tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "train_dataloader, val_dataloader = data_process(X_train, y_train, X_val, y_val, batch_size=32)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 42,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "fFfTQEHZ_Hc6",
        "outputId": "396ef346-4ab4-405d-ae9e-e1e4a760fe7d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.680220   |     -      |     -     |   5.93   \n",
            "   1    |   40    |   0.672429   |     -      |     -     |   5.72   \n",
            "   1    |   60    |   0.656117   |     -      |     -     |   5.84   \n",
            "   1    |   80    |   0.626486   |     -      |     -     |   5.93   \n",
            "   1    |   100   |   0.628667   |     -      |     -     |   6.03   \n",
            "   1    |   120   |   0.627794   |     -      |     -     |   6.09   \n",
            "   1    |   140   |   0.629517   |     -      |     -     |   6.06   \n",
            "   1    |   160   |   0.621178   |     -      |     -     |   5.94   \n",
            "   1    |   180   |   0.632668   |     -      |     -     |   5.85   \n",
            "   1    |   200   |   0.642153   |     -      |     -     |   5.79   \n",
            "   1    |   220   |   0.681749   |     -      |     -     |   5.75   \n",
            "   1    |   240   |   0.639850   |     -      |     -     |   5.68   \n",
            "   1    |   260   |   0.655336   |     -      |     -     |   5.68   \n",
            "   1    |   280   |   0.589119   |     -      |     -     |   5.68   \n",
            "   1    |   300   |   0.603616   |     -      |     -     |   5.70   \n",
            "   1    |   320   |   0.639547   |     -      |     -     |   5.73   \n",
            "   1    |   340   |   0.625050   |     -      |     -     |   5.76   \n",
            "   1    |   360   |   0.611637   |     -      |     -     |   5.80   \n",
            "   1    |   380   |   0.607564   |     -      |     -     |   5.83   \n",
            "   1    |   400   |   0.610650   |     -      |     -     |   5.85   \n",
            "   1    |   420   |   0.657249   |     -      |     -     |   5.86   \n",
            "   1    |   440   |   0.617021   |     -      |     -     |   5.86   \n",
            "   1    |   460   |   0.609914   |     -      |     -     |   5.88   \n",
            "   1    |   480   |   0.658771   |     -      |     -     |   5.86   \n",
            "   1    |   500   |   0.599964   |     -      |     -     |   5.85   \n",
            "   1    |   520   |   0.608690   |     -      |     -     |   5.81   \n",
            "   1    |   540   |   0.603555   |     -      |     -     |   5.81   \n",
            "   1    |   560   |   0.601296   |     -      |     -     |   5.78   \n",
            "   1    |   580   |   0.571308   |     -      |     -     |   5.77   \n",
            "   1    |   600   |   0.615555   |     -      |     -     |   5.77   \n",
            "   1    |   620   |   0.559117   |     -      |     -     |   5.78   \n",
            "   1    |   640   |   0.596055   |     -      |     -     |   5.79   \n",
            "   1    |   660   |   0.608022   |     -      |     -     |   5.81   \n",
            "   1    |   680   |   0.579807   |     -      |     -     |   5.81   \n",
            "   1    |   700   |   0.601439   |     -      |     -     |   5.82   \n",
            "   1    |   720   |   0.561156   |     -      |     -     |   5.83   \n",
            "   1    |   725   |   0.584434   |     -      |     -     |   1.24   \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.620121   |  0.979687  |   24.39   |  228.08  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.626108   |     -      |     -     |   6.13   \n",
            "   2    |   40    |   0.600435   |     -      |     -     |   5.80   \n",
            "   2    |   60    |   0.630901   |     -      |     -     |   5.81   \n",
            "   2    |   80    |   0.568282   |     -      |     -     |   5.81   \n",
            "   2    |   100   |   0.545557   |     -      |     -     |   5.80   \n",
            "   2    |   120   |   0.550756   |     -      |     -     |   5.80   \n",
            "   2    |   140   |   0.570080   |     -      |     -     |   5.80   \n",
            "   2    |   160   |   0.603213   |     -      |     -     |   5.81   \n",
            "   2    |   180   |   0.534509   |     -      |     -     |   5.80   \n",
            "   2    |   200   |   0.601349   |     -      |     -     |   5.81   \n",
            "   2    |   220   |   0.580196   |     -      |     -     |   5.80   \n",
            "   2    |   240   |   0.562656   |     -      |     -     |   5.81   \n",
            "   2    |   260   |   0.545589   |     -      |     -     |   5.81   \n",
            "   2    |   280   |   0.599140   |     -      |     -     |   5.81   \n",
            "   2    |   300   |   0.573074   |     -      |     -     |   5.82   \n",
            "   2    |   320   |   0.590288   |     -      |     -     |   5.83   \n",
            "   2    |   340   |   0.571498   |     -      |     -     |   5.83   \n",
            "   2    |   360   |   0.610174   |     -      |     -     |   5.83   \n",
            "   2    |   380   |   0.566455   |     -      |     -     |   5.82   \n",
            "   2    |   400   |   0.518583   |     -      |     -     |   5.81   \n",
            "   2    |   420   |   0.613993   |     -      |     -     |   5.82   \n",
            "   2    |   440   |   0.557245   |     -      |     -     |   5.81   \n",
            "   2    |   460   |   0.594764   |     -      |     -     |   5.79   \n",
            "   2    |   480   |   0.585019   |     -      |     -     |   5.79   \n",
            "   2    |   500   |   0.601024   |     -      |     -     |   5.78   \n",
            "   2    |   520   |   0.543132   |     -      |     -     |   5.79   \n",
            "   2    |   540   |   0.554609   |     -      |     -     |   5.77   \n",
            "   2    |   560   |   0.558607   |     -      |     -     |   5.78   \n",
            "   2    |   580   |   0.517307   |     -      |     -     |   5.77   \n",
            "   2    |   600   |   0.562095   |     -      |     -     |   5.78   \n",
            "   2    |   620   |   0.537070   |     -      |     -     |   5.79   \n",
            "   2    |   640   |   0.536123   |     -      |     -     |   5.79   \n",
            "   2    |   660   |   0.534128   |     -      |     -     |   5.79   \n",
            "   2    |   680   |   0.542564   |     -      |     -     |   5.81   \n",
            "   2    |   700   |   0.571475   |     -      |     -     |   5.81   \n",
            "   2    |   720   |   0.573182   |     -      |     -     |   5.82   \n",
            "   2    |   725   |   0.647198   |     -      |     -     |   1.24   \n",
            "----------------------------------------------------------------------\n",
            "   2    |    -    |   0.570917   |  0.979527  |   25.20   |  227.53  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   3    |   20    |   0.540449   |     -      |     -     |   6.08   \n",
            "   3    |   40    |   0.544896   |     -      |     -     |   5.80   \n",
            "   3    |   60    |   0.531846   |     -      |     -     |   5.82   \n",
            "   3    |   80    |   0.566254   |     -      |     -     |   5.82   \n",
            "   3    |   100   |   0.562637   |     -      |     -     |   5.82   \n",
            "   3    |   120   |   0.540052   |     -      |     -     |   5.83   \n",
            "   3    |   140   |   0.490892   |     -      |     -     |   5.83   \n",
            "   3    |   160   |   0.572238   |     -      |     -     |   5.83   \n",
            "   3    |   180   |   0.571856   |     -      |     -     |   5.82   \n",
            "   3    |   200   |   0.555745   |     -      |     -     |   5.81   \n",
            "   3    |   220   |   0.540868   |     -      |     -     |   5.81   \n",
            "   3    |   240   |   0.557190   |     -      |     -     |   5.80   \n",
            "   3    |   260   |   0.565202   |     -      |     -     |   5.80   \n",
            "   3    |   280   |   0.574913   |     -      |     -     |   5.80   \n",
            "   3    |   300   |   0.568980   |     -      |     -     |   5.78   \n",
            "   3    |   320   |   0.518706   |     -      |     -     |   5.78   \n",
            "   3    |   340   |   0.526547   |     -      |     -     |   5.78   \n",
            "   3    |   360   |   0.526632   |     -      |     -     |   5.78   \n",
            "   3    |   380   |   0.559919   |     -      |     -     |   5.78   \n",
            "   3    |   400   |   0.585993   |     -      |     -     |   5.77   \n",
            "   3    |   420   |   0.508563   |     -      |     -     |   5.78   \n",
            "   3    |   440   |   0.567122   |     -      |     -     |   5.78   \n",
            "   3    |   460   |   0.545561   |     -      |     -     |   5.79   \n",
            "   3    |   480   |   0.558752   |     -      |     -     |   5.78   \n",
            "   3    |   500   |   0.543586   |     -      |     -     |   5.78   \n",
            "   3    |   520   |   0.529595   |     -      |     -     |   5.79   \n",
            "   3    |   540   |   0.503991   |     -      |     -     |   5.80   \n",
            "   3    |   560   |   0.543104   |     -      |     -     |   5.79   \n",
            "   3    |   580   |   0.511540   |     -      |     -     |   5.78   \n",
            "   3    |   600   |   0.520930   |     -      |     -     |   5.78   \n",
            "   3    |   620   |   0.542977   |     -      |     -     |   5.79   \n",
            "   3    |   640   |   0.536020   |     -      |     -     |   5.78   \n",
            "   3    |   660   |   0.544765   |     -      |     -     |   5.78   \n",
            "   3    |   680   |   0.570111   |     -      |     -     |   5.78   \n",
            "   3    |   700   |   0.541559   |     -      |     -     |   5.80   \n",
            "   3    |   720   |   0.528526   |     -      |     -     |   5.79   \n",
            "   3    |   725   |   0.508780   |     -      |     -     |   1.23   \n",
            "----------------------------------------------------------------------\n",
            "   3    |    -    |   0.544153   |  1.011360  |   25.41   |  227.15  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Training complete!\n"
          ]
        }
      ],
      "source": [
        "set_seed(42)    # Set seed for reproducibility\n",
        "bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=4)\n",
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=3, evaluation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "lyxq0fPUvbDA",
        "outputId": "df940a5d-5951-4b9e-bab1-81cfdf893d1a"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.545204   |     -      |     -     |   6.37   \n",
            "   1    |   40    |   0.519625   |     -      |     -     |   6.19   \n",
            "   1    |   60    |   0.574206   |     -      |     -     |   6.25   \n",
            "   1    |   80    |   0.485586   |     -      |     -     |   6.09   \n",
            "   1    |   100   |   0.471760   |     -      |     -     |   5.96   \n",
            "   1    |   120   |   0.480813   |     -      |     -     |   5.84   \n",
            "   1    |   140   |   0.518540   |     -      |     -     |   5.75   \n",
            "   1    |   160   |   0.546737   |     -      |     -     |   5.70   \n",
            "   1    |   180   |   0.462260   |     -      |     -     |   5.68   \n",
            "   1    |   200   |   0.541311   |     -      |     -     |   5.68   \n",
            "   1    |   220   |   0.502894   |     -      |     -     |   5.68   \n",
            "   1    |   240   |   0.497534   |     -      |     -     |   5.69   \n",
            "   1    |   260   |   0.490589   |     -      |     -     |   5.75   \n",
            "   1    |   280   |   0.570047   |     -      |     -     |   5.79   \n",
            "   1    |   300   |   0.512959   |     -      |     -     |   5.85   \n",
            "   1    |   320   |   0.519200   |     -      |     -     |   5.87   \n",
            "   1    |   340   |   0.495737   |     -      |     -     |   5.91   \n",
            "   1    |   360   |   0.560948   |     -      |     -     |   5.89   \n",
            "   1    |   380   |   0.471855   |     -      |     -     |   5.88   \n",
            "   1    |   400   |   0.448698   |     -      |     -     |   5.84   \n",
            "   1    |   420   |   0.552268   |     -      |     -     |   5.81   \n",
            "   1    |   440   |   0.488916   |     -      |     -     |   5.78   \n",
            "   1    |   460   |   0.506677   |     -      |     -     |   5.77   \n",
            "   1    |   480   |   0.496571   |     -      |     -     |   5.76   \n",
            "   1    |   500   |   0.544745   |     -      |     -     |   5.76   \n",
            "   1    |   520   |   0.478218   |     -      |     -     |   5.76   \n",
            "   1    |   540   |   0.488052   |     -      |     -     |   5.78   \n",
            "   1    |   560   |   0.490043   |     -      |     -     |   5.80   \n",
            "   1    |   580   |   0.439320   |     -      |     -     |   5.81   \n",
            "   1    |   600   |   0.461959   |     -      |     -     |   5.82   \n",
            "   1    |   620   |   0.452706   |     -      |     -     |   5.82   \n",
            "   1    |   640   |   0.461927   |     -      |     -     |   5.83   \n",
            "   1    |   660   |   0.477887   |     -      |     -     |   5.82   \n",
            "   1    |   680   |   0.471521   |     -      |     -     |   5.82   \n",
            "   1    |   700   |   0.499008   |     -      |     -     |   5.81   \n",
            "   1    |   720   |   0.518353   |     -      |     -     |   5.81   \n",
            "   1    |   725   |   0.644334   |     -      |     -     |   1.24   \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.502287   |  1.099083  |   29.30   |  228.67  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.479828   |     -      |     -     |   6.07   \n",
            "   2    |   40    |   0.473536   |     -      |     -     |   5.79   \n",
            "   2    |   60    |   0.434656   |     -      |     -     |   5.79   \n",
            "   2    |   80    |   0.495029   |     -      |     -     |   5.80   \n",
            "   2    |   100   |   0.500219   |     -      |     -     |   5.81   \n",
            "   2    |   120   |   0.467589   |     -      |     -     |   5.80   \n",
            "   2    |   140   |   0.403112   |     -      |     -     |   5.81   \n",
            "   2    |   160   |   0.528798   |     -      |     -     |   5.81   \n",
            "   2    |   180   |   0.532970   |     -      |     -     |   5.82   \n",
            "   2    |   200   |   0.482465   |     -      |     -     |   5.82   \n",
            "   2    |   220   |   0.485707   |     -      |     -     |   5.82   \n",
            "   2    |   240   |   0.494418   |     -      |     -     |   5.82   \n",
            "   2    |   260   |   0.499734   |     -      |     -     |   5.82   \n",
            "   2    |   280   |   0.516807   |     -      |     -     |   5.82   \n",
            "   2    |   300   |   0.500395   |     -      |     -     |   5.81   \n",
            "   2    |   320   |   0.442923   |     -      |     -     |   5.81   \n",
            "   2    |   340   |   0.443293   |     -      |     -     |   5.81   \n",
            "   2    |   360   |   0.460397   |     -      |     -     |   5.82   \n",
            "   2    |   380   |   0.473031   |     -      |     -     |   5.81   \n",
            "   2    |   400   |   0.522881   |     -      |     -     |   5.81   \n",
            "   2    |   420   |   0.444609   |     -      |     -     |   5.81   \n",
            "   2    |   440   |   0.505379   |     -      |     -     |   5.81   \n",
            "   2    |   460   |   0.449637   |     -      |     -     |   5.81   \n",
            "   2    |   480   |   0.506319   |     -      |     -     |   5.80   \n",
            "   2    |   500   |   0.485601   |     -      |     -     |   5.81   \n",
            "   2    |   520   |   0.458831   |     -      |     -     |   5.81   \n",
            "   2    |   540   |   0.439167   |     -      |     -     |   5.81   \n",
            "   2    |   560   |   0.492297   |     -      |     -     |   5.81   \n",
            "   2    |   580   |   0.432948   |     -      |     -     |   5.80   \n",
            "   2    |   600   |   0.464486   |     -      |     -     |   5.81   \n",
            "   2    |   620   |   0.459979   |     -      |     -     |   5.81   \n",
            "   2    |   640   |   0.474550   |     -      |     -     |   5.81   \n",
            "   2    |   660   |   0.473820   |     -      |     -     |   5.81   \n",
            "   2    |   680   |   0.516878   |     -      |     -     |   5.81   \n",
            "   2    |   700   |   0.474943   |     -      |     -     |   5.80   \n",
            "   2    |   720   |   0.476459   |     -      |     -     |   5.81   \n",
            "   2    |   725   |   0.423572   |     -      |     -     |   1.24   \n",
            "----------------------------------------------------------------------\n",
            "   2    |    -    |   0.477234   |  1.175859  |   29.30   |  227.64  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Counter 1 of 5\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   3    |   20    |   0.484467   |     -      |     -     |   6.10   \n",
            "   3    |   40    |   0.453539   |     -      |     -     |   5.81   \n",
            "   3    |   60    |   0.456521   |     -      |     -     |   5.80   \n",
            "   3    |   80    |   0.522426   |     -      |     -     |   5.80   \n",
            "   3    |   100   |   0.486579   |     -      |     -     |   5.80   \n",
            "   3    |   120   |   0.457473   |     -      |     -     |   5.79   \n",
            "   3    |   140   |   0.454332   |     -      |     -     |   5.81   \n",
            "   3    |   160   |   0.526909   |     -      |     -     |   5.80   \n",
            "   3    |   180   |   0.440170   |     -      |     -     |   5.80   \n",
            "   3    |   200   |   0.498661   |     -      |     -     |   5.79   \n",
            "   3    |   220   |   0.480618   |     -      |     -     |   5.80   \n",
            "   3    |   240   |   0.437926   |     -      |     -     |   5.79   \n",
            "   3    |   260   |   0.509216   |     -      |     -     |   5.79   \n",
            "   3    |   280   |   0.430851   |     -      |     -     |   5.80   \n",
            "   3    |   300   |   0.430688   |     -      |     -     |   5.80   \n",
            "   3    |   320   |   0.471829   |     -      |     -     |   5.79   \n",
            "   3    |   340   |   0.504197   |     -      |     -     |   5.79   \n",
            "   3    |   360   |   0.434864   |     -      |     -     |   5.78   \n",
            "   3    |   380   |   0.470715   |     -      |     -     |   5.79   \n",
            "   3    |   400   |   0.482817   |     -      |     -     |   5.80   \n",
            "   3    |   420   |   0.450678   |     -      |     -     |   5.79   \n",
            "   3    |   440   |   0.449919   |     -      |     -     |   5.80   \n",
            "   3    |   460   |   0.464270   |     -      |     -     |   5.79   \n",
            "   3    |   480   |   0.530542   |     -      |     -     |   5.79   \n",
            "   3    |   500   |   0.509119   |     -      |     -     |   5.80   \n",
            "   3    |   520   |   0.463976   |     -      |     -     |   5.81   \n",
            "   3    |   540   |   0.445084   |     -      |     -     |   5.80   \n",
            "   3    |   560   |   0.420181   |     -      |     -     |   5.80   \n",
            "   3    |   580   |   0.425177   |     -      |     -     |   5.80   \n",
            "   3    |   600   |   0.493739   |     -      |     -     |   5.80   \n",
            "   3    |   620   |   0.513290   |     -      |     -     |   5.80   \n",
            "   3    |   640   |   0.406431   |     -      |     -     |   5.80   \n",
            "   3    |   660   |   0.489082   |     -      |     -     |   5.80   \n",
            "   3    |   680   |   0.416472   |     -      |     -     |   5.80   \n",
            "   3    |   700   |   0.416878   |     -      |     -     |   5.79   \n",
            "   3    |   720   |   0.468184   |     -      |     -     |   5.79   \n",
            "   3    |   725   |   0.483141   |     -      |     -     |   1.24   \n",
            "----------------------------------------------------------------------\n",
            "   3    |    -    |   0.466745   |  1.195038  |   29.30   |  227.24  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Counter 2 of 5\n",
            "Early stopping with best_acc:  29.30327868852459 and val_acc for this epoch:  29.30327868852459 ...\n",
            "Training complete!\n"
          ]
        }
      ],
      "source": [
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=3, evaluation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "BwDvSUU-GrFW"
      },
      "outputs": [],
      "source": [
        "torch.save(bert_classifier.state_dict(), './Model/BERT_raw_to_fine_tune_ord4.pt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 330
        },
        "id": "16ynZzt_A95T",
        "outputId": "4ca200ac-fb0b-4323-d0aa-31b79eaa888b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AUC: 0.7044\n",
            "Accuracy: 45.36%\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debxV8/rA8c9TmqRCXEODotAgDUeZGkgkEbekSCJllvHecO9F1zVcLpd7M1S6+RkKuRJKLk1Co9KoQVEnRVJIg4bn98d3nc7qOHufdc7Za689PO/Xa7/OHtZe+9nrnLOf/f1+1/f5iqpijDHGxFIm6gCMMcakNksUxhhj4rJEYYwxJi5LFMYYY+KyRGGMMSYuSxTGGGPiskRhikVEFolIu6jjSBUicreIDIvotUeIyANRvHaiichlIvJ+CZ9rf5Mhs0SRxkTkKxHZJiJbRGS998FxQJivqaqNVHVymK+RR0QqiMhDIrLae5/LReROEZFkvH4h8bQTkVz/far6oKpeHdLriYjcLCILReQXEckVkddF5IQwXq+kROQ+EXmpNPtQ1ZdV9ewAr/Wb5JjMv8lsZYki/Z2vqgcATYFmwF0Rx1NsIrJfjIdeB9oDnYAqwOVAf+DJEGIQEUm1/4cngQHAzcDBwLHAGOC8RL9QnN9B6KJ8bROQqtolTS/AV8BZvtt/B9713T4Z+ATYDHwOtPM9djDwH+AbYBMwxvdYZ2Ce97xPgCYFXxM4EtgGHOx7rBnwPVDOu30VsMTb/wTgKN+2CtwALAdWFfLe2gPbgVoF7m8F7AbqebcnAw8BM4GfgLcKxBTvGEwG/gZ87L2XesCVXsw/AyuBa7xtK3vb7AG2eJcjgfuAl7xt6njv6wpgtXcs7vG9XiXgBe94LAH+AOTG+N3W995nyzi//xHAYOBdL94ZwDG+x58E1njHZQ7Q2vfYfcBo4CXv8auBlsCn3rFaB/wbKO97TiPgf8APwLfA3UBH4Fdgp3dMPve2rQY87+1nLfAAUNZ7rI93zJ8ANnqP9QGmeY+L99h3XmwLgMa4Lwk7vdfbArxd8P8AKOvF9aV3TOZQ4G/ILiX4rIk6ALuU4pe37z9ITe8f6knvdg3vn7ATruXYwbt9qPf4u8CrwEFAOaCtd38z7x+0lfdPd4X3OhUKec2JQD9fPI8Cz3rXuwArgAbAfsCfgE9826r3oXMwUKmQ9/YwMCXG+/6a/A/wyd4HUWPch/kb5H9wF3UMJuM+0Bt5MZbDfVs/xvuwagtsBZp727ejwAc7hSeKobikcCKwA2jgf0/eMa8JzC+4P99+rwW+LuL3P8J7Py29+F8GRvke7wVU9x67HVgPVPTFvRO40Ds2lYAWuMS6n/delgC3eNtXwX3o3w5U9G63KngMfK/9JvCc9zv5HS6R5/3O+gC7gJu816rEvoniHNwH/IHe76EBcITvPT8Q5//gTtz/wXHec08Eqkf9v5rul8gDsEspfnnuH2QL7puTAh8CB3qP/RF4scD2E3Af/EfgvhkfVMg+nwH+WuC+peQnEv8/5dXARO+64L69tvFujwf6+vZRBvehe5R3W4Ez47y3Yf4PvQKPTcf7po77sH/Y91hD3DfOsvGOge+5g4o4xmOAAd71dgRLFDV9j88EenjXVwLn+B67uuD+fI/dA0wvIrYRwDDf7U7AF3G23wSc6It7ahH7vwV407veE5gbY7u9x8C7fRguQVby3dcTmORd7wOsLrCPPuQnijOBZbikVaaQ9xwvUSwFuoTx/5bNl1TrkzXFd6GqVsF9iB0PHOLdfxRwsYhszrsAp+OSRC3gB1XdVMj+jgJuL/C8WrhuloLeAE4RkSOANrjk85FvP0/69vEDLpnU8D1/TZz39b0Xa2GO8B4vbD9f41oGhxD/GBQag4icKyLTReQHb/tO5B/ToNb7rm8F8k4wOLLA68V7/xuJ/f6DvBYicoeILBGRH733Uo1930vB936siLzjnRjxE/Cgb/tauO6cII7C/Q7W+Y77c7iWRaGv7aeqE3HdXoOB70RkiIhUDfjaxYnTBGSJIkOo6hTct63HvLvW4L5NH+i7VFbVh73HDhaRAwvZ1RrgbwWet7+qjizkNTcB7wOXAJfiWgDq2881BfZTSVU/8e8izlv6AGglIrX8d4pIK9yHwUTf3f5tauO6VL4v4hj8JgYRqYBLfo8Bh6nqgcA4XIIrKt4g1uG6nAqLu6APgZoiklOSFxKR1rgxkO64luOBwI/kvxf47ft5BvgCqK+qVXF9/XnbrwGOjvFyBfezBteiOMR33KuqaqM4z9l3h6pPqWoLXAvxWFyXUpHP8177mCK2McVkiSKz/BPoICIn4gYpzxeRc0SkrIhU9E7vrKmq63BdQ0+LyEEiUk5E2nj7GApcKyKtvDOBKovIeSJSJcZrvgL0Brp51/M8C9wlIo0ARKSaiFwc9I2o6ge4D8s3RKSR9x5O9t7XM6q63Ld5LxFpKCL7A4OA0aq6O94xiPGy5YEKwAZgl4icC/hP2fwWqC4i1YK+jwJewx2Tg0SkBnBjrA299/c0MNKLubwXfw8RGRjgtargxgE2APuJyF+Aor6VV8ENHm8RkeOB63yPvQMcISK3eKctV/GSNrjjUifvrDHv7+t94B8iUlVEyojIMSLSNkDciMhJ3t9fOeAX3EkNe3yvFSthgeuy/KuI1Pf+fpuISPUgr2tis0SRQVR1A/B/wF9UdQ1uQPlu3IfFGty3srzf+eW4b95f4Aavb/H2MRvoh2v6b8INSPeJ87JjcWforFfVz32xvAk8AozyujEWAucW8y11BSYB7+HGYl7CnUlzU4HtXsS1ptbjBlpv9mIo6hjsQ1V/9p77Gu69X+q9v7zHvwBGAiu9LpXCuuPiGQTkAqtwLabRuG/esdxMfhfMZlyXykXA2wFeawLuuC3DdcdtJ35XF8AduPf8M+4Lw6t5D3jHpgNwPu44LwfO8B5+3fu5UUQ+8673xiXexbhjOZpgXWngEtpQ73lf47rhHvUeex5o6B3/MYU893Hc7+99XNJ7HjdYbkpB8nsKjEk/IjIZN5Aayezo0hCR63AD3YG+aRsTFWtRGJMkInKEiJzmdcUchzvV9M2o4zKmKKElChEZLiLficjCGI+LiDwlIitEZL6INA8rFmNSRHnc2T8/4wbj38KNQxiT0kLrevIGR7cA/6eqjQt5vBOur7kTbnLXk6raquB2xhhjohVai0JVp+LOnY+lCy6JqKpOBw70zsc3xhiTQqIsxlWDfc/CyPXuW1dwQxHpj6vzQuXKlVscf/zxSQnQGGOisGED/BDva7bPli3u5wEx6kYftuNrDti1mc911/eqemhJ4kmLqo2qOgQYApCTk6OzZ8+OOCJjjEmsIUPgFW8m0pw57mfbgOfDXXop9O/vuyNvSEEEnnkGvvsOue++r0saW5SJYi37zkyt6d1njDEZy58Q/KZMcT/btnWX33z4B7V2LVx3HVxyCVx2mbsOcN99JQ050kQxFrhRREbhBrN/9GZ0GmNMyoj1wV5S/oTgV6rkAK4VMWwY3HEH7NwJ5yVu2ZLQEoWIjMQVqjtE3Kpg9+IKhaGqz+Jq6HTCzfzdilsHwBhjUsorr8C8edC0aWL2V+qEUJgvv4R+/WDSJDjjDBg6FI5JXMmr0BKFqvYs4nHFLVxjjDEpxd+KyEsSkydHGlJ8Cxa4gY0hQ+Dqq93YRAKlxWC2McYkSpCuJH/3UNOmrgWQchYuhM8+g9694cILYeVKqB5O/UNLFMaYjOdPDrHGCPxC6R5KlF9/hQcfdJfDDoPu3aFixdCSBFiiMMZkqFjJIaWTQFFmzIC+fWHRIujVC554wiWJkFmiMMakvcK6kzImOeRZuxZat3atiHfeSehZTUWxRGGMSXuFnZmUEckBYNkyOPZYqFEDXn0V2reHqkFXhk0MSxTGmJQWZPA5Lc5MKq7Nm+EPf3BzIyZPhjZt4KKLIgnFEoUxJiUEmbEcS8qemVRSY8e6GdXr18Odd8JJJ0UajiUKY0xKiDWxLWO6kIK6+mp4/nk44QR46y3IyYk6IksUxpjUkXHdR0H5i/jl5MBRR8Ef/wjly0cbl8cShTEmMoXNgM46a9bAtddCjx5w+eXueoqxNbONMUk3ZAi0awfXXJM/BpFx4wxF2bPHlQBv1Mg1o3bsiDqimKxFYYxJurzxiKwbf8izfLkbi5g6Fc46y2XOunWjjiomSxTGmKRIu0J7YVq8GObPh+HDoU+fhBfxSzTrejLGJEVeKwKysJsJ4PPP4YUX3PUuXVwRvyuvTPkkAdaiMMaEyFoRuLGHBx6Ahx+GI45wK89VrAgHHRR1ZIFZi8IYE5qsb0V8+ik0a+YSxaWXwty5SSnil2jWojDGhCorWxHgivi1bQuHHw7jxsG550YdUYlZi8IYYxJpyRL3s0YNeO01VxI8jZMEWIvCGJMAseo0ZdUkuk2b4Pbb4T//cae9tm7tVp7LAJYojDFxFXfpUL+sGZd48024/nrYsAHuuivyIn6JZonCGLNXUQsAxZK1E+cArrrKtSKaNoV334XmzaOOKOEsURhj9sroBYASyV/E7+SToX59uOMOKFcu2rhCYonCmCxU1JhCVp6lFNTXX7siVZdeCr17Z0UGtbOejMlC/vkNflkzplASe/bA4MHQuDFMmwY7d0YdUdJYi8KYLGUth2JYutQV8Zs2Dc4+G557DurUiTqqpLFEYUyGKc4a0yagpUvdfIgRI1x3UxrUZ0ok63oyJsPE6lbysy6mAObOdWczAVxwgSvid8UVWZckwFoUxmQk61Yqhe3bYdAg+Pvf3ezqnj1dfaYDD4w6sshYi8IYY/J8/LHLsg895LqY5s1LyyJ+iWYtCmMygK09nQBr18IZZ7hWxIQJbtDaAJYojEkrsQaq/bOnbfyhmBYvhoYNXYJ44w2XLA44IOqoUoolCmPSQF6CiFVOw2ZPl8APP8Btt7lV56ZMgTZt4Pzzo44qJVmiMCZF+VsP/gRhCSEB3ngDbrgBNm6Ee+6Bli2jjiilWaIwJmJBupMsQSRQnz6uFdG8Obz3ng3oBGCJwpiIFVaIDyw5JJS/iN+pp0KDBm7tiP3sIzCIUI+SiHQEngTKAsNU9eECj9cGXgAO9LYZqKrjwozJmFRk8x5CtGqVy7a9erkJc5Z5iy20eRQiUhYYDJwLNAR6ikjDApv9CXhNVZsBPYCnw4rHmFQyZAi0a+cuRc2iNiW0ezc89ZQr4jd9en6rwhRbmBPuWgIrVHWlqv4KjAK6FNhGgare9WrANyHGY0zK8JfZsNNZQ7BkiVuKdMAA14e3aJEbmzAlEmbXUw1gje92LtCqwDb3Ae+LyE1AZeCswnYkIv2B/gC1a9dOeKDGJENhk+KsuykkK1a4Qn4vvgiXXZaV9ZkSKeoSHj2BEapaE+gEvCgiv4lJVYeoao6q5hx66KFJD9KYRLBWRMjmzIHhw9318893YxO9elmSSIAwWxRrgVq+2zW9+/z6Ah0BVPVTEakIHAJ8F2JcxkTGWhEh2LYN7r8fHnsMatVyGbhiRahatejnmkDCTBSzgPoiUheXIHoABb9DrQbaAyNEpAFQEdgQYkzGlEiQNR6KYjWYQjB1qltQaPly6NvXJQsr4pdwoSUKVd0lIjcCE3Cnvg5X1UUiMgiYrapjgduBoSJyK25gu4+qnZpgUkOsmdElZd1NCbZ2LbRv71oRH3zgrptQSLp9Lufk5Ojs2bOjDsNkgbxTV/NaATb5LUUsWAAnnOCuv/OOK+JXuXK0MaUBEZmjqjklea5NSzQmDhtTSCHffw+33govvZRfxK9z56ijygpRn/VkTEqxiXApSBVee82VAh81Cu69F1oVPNPehMlaFCbrxRqLsDGFFHHFFW4+RE4OfPhhfreTSRpLFCbr+YvyWSG+FOEv4te2LTRpArfcYkX8ImJH3RhsLCKlrFwJ/fq5yXJXXulOezWRsjEKk5VsLCIF7d4N//yn61qaNQvK2MdTqrDfhMlKVk4jxSxeDKed5s5qOuMMd/uKK6KOynis68lkLetuSiGrVsGXX7oM3qOH1WdKMZYojDHRmDXLNev69YPzznNjE1WqRB2VKYR1PZmsYeMSKWLrVrjjDjj5ZHjoIdi+3d1vSSJlWaIwWcPGJVLA5MnuVNd//MO1JObOtSJ+acC6nkzGKKrCqy0WFLHcXOjQAY46CiZOdIPWJi1Yi8JkDH+LoTDWiojI55+7nzVrwltvwfz5liTSjLUoTEaxFkMK2bDBrVk9cqT7pbRtC506RR2VKQFrUZi0ZgPUKUjVJYeGDWH0aLf63CmnRB2VKQVrUZi0Y0X8Utzll8PLL7sKr88/D40aRR2RKaXAiUJE9lfVrWEGY0wQVsQvBe3Z4ybJibjxhxYt4OaboWzZqCMzCVBkohCRU4FhwAFAbRE5EbhGVa8POzhjYrGxiBSyYoU71fXyy+Gqq6yIXwYKMkbxBHAOsBFAVT8H2oQZlDEF2VhECtq1Cx57zBXxmzsXypePOiITkkBdT6q6RvatvbI7nHCMyWdjESls4UJXAnz2bOjSBZ5+Go48MuqoTEiCJIo1XveTikg5YACwJNywjLGxiJS2ejV8/bVbmrR7dyvil+GCJIprgSeBGsBa4H3AxidMUthYRAqZMcNNnuvf382HWLkSDjgg6qhMEgRJFMep6mX+O0TkNODjcEIy2SZW6Y281oSJ2C+/wJ//7BYVOvpot05EhQqWJLJIkMHsfwW8z5gSiVV6w8YiUsDEia6I3xNPwLXXwmefuSRhskrMFoWInAKcChwqIrf5HqoK2MnRJqGsiykF5ebCOedA3brubII2drJjtorX9VQeN3diP8BfKP4noFuYQRljIjR3LjRr5or4vf22O5OgUqWoozIRipkoVHUKMEVERqjq10mMyRgThW+/dbOpX3stv4hfx45RR2VSQJDB7K0i8ijQCNi7woiqnhlaVMaY5FF1tZkGDIAtW+CBB+DUU6OOyqSQIIPZLwNfAHWB+4GvgFkhxmSMSaZLL3XlN447zp1VcM89UK5c1FGZFBKkRVFdVZ8XkQG+7ihLFKZECjsV1k6DjYC/iN/ZZ7sy4DfcYEX8TKGCtCh2ej/Xich5ItIMODjEmEwGK+xUWDsNNsmWLXMVXocPd7evvNIqvZq4grQoHhCRasDtuPkTVYFbQo3KZBR/K8LWrY7Qrl3w+ONw771QsaKdyWQCKzJRqOo73tUfgTNg78xsYwLx12yy1kNE5s93JcDnzIGLLoLBg+GII6KOyqSJeBPuygLdcTWe3lPVhSLSGbgbqAQ0S06IJhNYKyJiubmwZg28/jp07WpF/EyxxBujeB64GqgOPCUiLwGPAX9X1UBJQkQ6ishSEVkhIgNjbNNdRBaLyCIRKaTijzGmRD75BJ591l3PK+LXrZslCVNs8bqecoAmqrpHRCoC64FjVHVjkB17LZLBQAcgF5glImNVdbFvm/rAXcBpqrpJRH5X0jdijPFs2eJOcf3Xv+CYY9xgdYUKULly1JGZNBWvRfGrqu4BUNXtwMqgScLTElihqitV9VdgFNClwDb9gMGqusl7ne+KsX9jTEHvvw+NG7skccMNVsTPJES8FsXxIjLfuy7AMd5tAVRVmxSx7xrAGt/tXKBVgW2OBRCRj3GFBu9T1fcK7khE+gP9AWrXrl3EyxqTpdasgfPOc62IqVPh9NOjjshkiHiJokGSXr8+0A6oCUwVkRNUdbN/I1UdAgwByMnJ0STEZUz6mDMHWrSAWrVg3Dho3dqd/mpMgsTselLVr+NdAux7LVDLd7umd59fLjBWVXeq6ipgGS5xGGOKsn49XHwx5OTkLyreoYMlCZNwQWZml9QsoL6I1BWR8kAPYGyBbcbgWhOIyCG4rqiVIcZkTPpThRdegIYNXRnwBx+0In4mVEFmZpeIqu4SkRuBCbjxh+GqukhEBgGzVXWs99jZIrIY2A3cWcwBc2OyT48erhT4aafBsGFw/PFRR2QyXKBEISKVgNqqurQ4O1fVccC4Avf9xXddgdu8izEmFn8Rv06d3DjE9ddDmTA7BYxxivwrE5HzgXnAe97tpiJSsAvJGBOWL75wy5A+/7y7fcUVcOONliRM0gT5S7sPNydiM4CqzsOtTWGMCdPOnW784cQTYfFiOOCAqCMyWSpI19NOVf1R9p32b6eoGhOmefPcjOp581zZjX/9Cw4/POqoTJYKkigWicilQFmv5MbNwCfhhmVMllu/3l3eeAN+//uoozFZLkjX00249bJ3AK/gyo3behTGJNq0afD00+56x47w5ZeWJExKCNKiOF5V7wHuCTsYk94KW+YUbKnTIv38M9x1l1sjon596NvX1Wfaf/+oIzMGCNai+IeILBGRv4pI49AjMmmrsGVOwRYrimvCBFfE7+mnYcAAK+JnUlKQFe7OEJHDcYsYPSciVYFXVfWB0KMzKc+WOS2FNWugc2eoV891O9nsapOiAp2IrarrVfUp4FrcnIq/FPEUkyX8rQhrOQSgCjNnuuu1asH48TB3riUJk9KKbFGISAPgEqArsBF4Fbg95LhMGrFWREDr1rk1It580x2wtm3hrLOijsqYIgUZzB6OSw7nqOo3IcdjTOZRhREj4LbbYPt2eOQRV6fJmDQRZIzilGQEYkzG6t4dRo929ZmGDYNjj406ImOKJWaiEJHXVLW7iCxg35nYQVe4MxmqsAFsU8Du3a6AX5kycP75cOaZcM01Vp/JpKV4LYoB3s/OyQjEpI+8AeymTW0Au1BLlri5EFdeCf36Qe/eUUdkTKnETBSqus67er2q/tH/mIg8Avzxt88ymcpOgw1g5043/vDXv7oCftWqRR2RMQkRpB3coZD7zk10ICb1DBkC7dq5yzXX5K+2aa2IQsyd65Yk/fOf4aKLXKuie/eoozImIeKNUVwHXA8cLSLzfQ9VAT4OOzATPX8XU9u2Ljn07x91VCnq22/h++9hzBjo0iXqaIxJqHhjFK8A44GHgIG++39W1R9CjcpEKq+bybqYijB1KixY4OZGdOwIK1ZApUpRR2VMwsVLFKqqX4nIDQUfEJGDLVlkFv8YRF4XU14rwhTw008wcCA884w71fXqq119JksSJkMV1aLoDMzBnR7rX7lIgaNDjMskmXUzBTRunBuw+eYbN4Fu0CAr4mcyXryznjp7P23Z0wxlZzIV05o1bvzhuOPcBLpWraKOyJikKPKsJxE5TUQqe9d7icjjIlI7/NBM2KygXwCqMH26u16rFrz/visFbknCZJEgp8c+A2wVkRNxxQC/BF4MNSqTNHmtiMmTravpN775Bi68EE45JX/g5owzoHz5aOMyJsmCJIpdqqpAF+DfqjoYd4qsMZlJ1dVkatjQtSAee8yK+JmsFqR67M8ichdwOdBaRMoA5cINy5RWrGVJ/axOUwzdusF//+tG9YcNcwsLGZPFgrQoLgF2AFep6nqgJvBoqFGZUou1LKmfjUv47N4Ne/a46xdeCM8+CxMnWpIwhmBlxteLyMvASSLSGZipqv8XfmimtOwspoAWLnRzIfr2dUX8Lr886oiMSSlBznrqDswELsatmz1DRLqFHZgpPn9tpqJaEwb49Ve4/35o3hy+/BIOOijqiIxJSUHGKO4BTlLV7wBE5FDgA2B0mIGZ4rPy38UwZw706eNaE5deCv/8Jxx6aNRRGZOSgiSKMnlJwrORYGMbJgLW3RTQxo2weTO8/TZ0tiVXjIknSKJ4T0QmACO925cA48ILyRQl1hlNdhZTESZNckX8br4Zzj4bli+HihWjjsqYlFdky0BV7wSeA5p4lyEFFzIyyRXrjCbrborhxx9dfaYzz3SF/HbscPdbkjAmkHjrUdQHHgOOARYAd6jq2mQFZuKzLqaA3n4brr0W1q+HO+5wg9dWxM+YYonXohgOvAN0xVWQ/VdSIjImUdasga5doXp1V6/p0Udh//2jjsqYtBNvjKKKqg71ri8Vkc+SEZApela1jUXEoQqffgqnnppfxO/UU60+kzGlEK9FUVFEmolIcxFpDlQqcLtIItJRRJaKyAoRGRhnu64ioiKSU9w3kClirU9dGBuLiCE3Fy64wNVlyjuA7dpZkjCmlOK1KNYBj/tur/fdVuDMeDsWkbLAYKADkAvMEpGxqrq4wHZVgAHAjOKFnlls4aBS2LMHhg6FO++EXbvg8cfh9NOjjsqYjBFv4aIzSrnvlsAKVV0JICKjcBVoFxfY7q/AI8CdpXy9tGcD1CXUtSuMGePOaho6FI62xReNSaQwJ87VANb4bud69+3ldWHVUtV34+1IRPqLyGwRmb1hw4bERxoRK7lRCrt25Rfx69rVJYgPPrAkYUwIIpth7ZUrfxy3GFJcqjpEVXNUNefQDCqzYCvMldD8+W4xoaHeuRa9ermifiLxn2eMKZEgM7NLai1Qy3e7pndfnipAY2CyuH/ww4GxInKBqs4OMa6kK2omtXU3BbRjBzz4oLscdJDVZjImSYJUjxVvrey/eLdri0jLAPueBdQXkboiUh7oAYzNe1BVf1TVQ1S1jqrWAaYDGZckwGZSJ8SsWa7K66BB0LMnLFkCv/991FEZkxWCtCieBvbgznIaBPwMvAGcFO9JqrpLRG4EJgBlgeGqukhEBgGzVXVsvOenI2s5hGjTJtiyBcaNg3PPjToaY7JKkETRSlWbi8hcAFXd5LUQiqSq4yhQQFBV/xJj23ZB9pnK/Ke4+lnLoYQmTnRF/AYMcEX8li2z8hvGRCBIotjpzYlQ2LsexZ5Qo0ozeS0JazkkyObNbk7EsGHQoIGr1VShgiUJYyISJFE8BbwJ/E5E/gZ0A/4UalQpKlbXUt4k4LyJcqYU3noLrrsOvv0W/vAHuO8+SxDGRCzImtkvi8gcoD0gwIWquiT0yFJQrK4lm0mdIKtXw8UXu1bE2LGQk7UVXYxJKUUmChGpDWwF3vbfp6qrwwwsVVnXUoKpwrRp0Lo11K7tJs2dfLLVZzImhQTpenoXNz4hQEWgLrAUaBRiXCYbrF7txh/Gj3fZt21baNMm6qiMMQUE6Xo6wX/bK7txfWgRpRj/uISV906QPXvg2Wfhj390LYqnnsPkorEAABUBSURBVLIifsaksGLPzFbVz0SkVRjBpAp/cvAPVNtprgny+9+7QesOHdzBrlMn6oiMMXEEGaO4zXezDNAc+Ca0iFKAlfwOwa5dUKaMu1xyCXTpAn36WH0mY9JAkBZFFd/1XbgxizfCCSd12KB1An3+OVx1FfTr58YkevaMOiJjTDHETRTeRLsqqnpHkuIxmWT7dnjgAXjkETj4YDj88KgjMsaUQMxEISL7efWaTktmQCZDzJwJV1wBX3zhfj7+uEsWxpi0E69FMRM3HjFPRMYCrwO/5D2oqv8NOTaTzn76CbZtg/feg3POiToaY0wpBBmjqAhsxFWPzZtPoYAlCrOv99+HRYvg1lvhrLNg6VIrv2FMBoiXKH7nnfG0kPwEkUdDjcqkl02b4LbbYMQIaNQIrr/eivgZk0HiLVxUFjjAu1TxXc+7ZBRbv7qE/vtfaNgQXnwR7roLZs+2BGFMhonXolinqoOSFknE/HMnbGJdQKtXQ48e0LixW1CoWbOoIzLGhCBeosi6mVA2dyIAVZg61c1ErF3bLS7UqhWUKxd1ZMaYkMTremqftCgiYt1NxfT1124Z0nbt8mubnH66JQljMlzMRKGqPyQzkCjkdTeBdTfFtWcP/PvfbqB62jT4179cWXBjTFYodlHATGPdTQFceCG8/babD/Hcc3DUUVFHZIxJoqxPFCaGnTuhbFlXxK9nT+jWDS6/3Ir4GZOF4o1RZCQblwjgs8+gZUu3ZgS4RNG7tyUJY7JU1iUKG5eIY9s2NxeiZUtYvx5q1Yo6ImNMCsjKricblyjE9OmueN+yZa4k+GOPwUEHRR2VMSYFZGWiMIX45Rc3LvG//7k6TcYY47FEkc3ee88V8bv9dmjf3pUEL18+6qiMMSkm68YoDLBxo+tmOvdceOEF+PVXd78lCWNMISxRZBNVGD3aFfF75RX4059g1ixLEMaYuKzrKZusXu1O82rSxK0dceKJUUdkjEkDWdOiyJs/kXVzJ1Rd4T5wM6onT3ZnOFmSMMYElDWJwl9GPGvmTqxaBWef7Qaq84r4nXoq7GcNSWNMcFn1iZE18yd273ZF/O6+25XheOYZK+JnjCmxrEoUWaNLF3j3XejUyZXhsBnWxphSyOhEMWSI63KC/G6njOUv4nf55a4+06WXWn0mY0yphTpGISIdRWSpiKwQkYGFPH6biCwWkfki8qGIJLR+ddbUdZo9G3JyXBcTwCWXwGWXWZIwxiREaC0KESkLDAY6ALnALBEZq6qLfZvNBXJUdauIXAf8HbgkkXFk9LjEtm1w332uLtNhh9k6EcaYUITZomgJrFDVlar6KzAK6OLfQFUnqepW7+Z0oGaI8WSWTz91p7j+/e+uiN/ixdC5c9RRGWMyUJhjFDWANb7buUCrONv3BcYX9oCI9Af6A9SuXTtR8aW3bdvcEqUffOBOfzXGmJCkxGC2iPQCcoC2hT2uqkOAIQA5OTmaxNBSy7hxrojfnXfCmWfCkiVQrlzUURljMlyYXU9rAf95mTW9+/YhImcB9wAXqOqOEONJX99/D716wXnnwcsv5xfxsyRhjEmCMBPFLKC+iNQVkfJAD2CsfwMRaQY8h0sS34UYS3pShVGjoEEDeO01uPdemDnTivgZY5IqtK4nVd0lIjcCE4CywHBVXSQig4DZqjoWeBQ4AHhd3Kmcq1X1grBiSjurV7ty4CeeCM8/DyecEHVExpgsFOoYhaqOA8YVuO8vvuu2lFpBqvDhh26VuaOOcjWaTjrJTaYzxpgIZE1RwLTw5ZfuDKYOHfKL+J18siUJY0ykLFGkgt274fHHXdfSnDnw3HNWxM8YkzJS4vTYrHf++TB+vJsw98wzUNPmHRpjUocliqj8+qtbF6JMGejTxxXy69HD6jMZY1KOdT1FYeZMaNECnn7a3e7e3VV7tSRhjElBliiSaetWuP12OOUU2LQJjjkm6oiMMaZI1vWULNOmuTkRK1fCNdfAI49AtWpRR2WMMUWyRJEseQsLTZoE7dpFHY0xxgRmiSJMb7/tCvf94Q9wxhmuFPh+dsiNMekl48YohgxxX9jbtctf3S7pNmxwy+ldcAGMHJlfxM+ShDEmDWVcooh0+VNVF0CDBjB6NAwaBDNmWBE/Y0xay8ivuJEtf7p6NVx5JTRr5or4NWoUQRDGGJNYGdeiSLo9e2DCBHf9qKPgo4/g448tSRhjMoYlitJYvtytNNexI0yd6u5r2dKK+BljMoolipLYtQsefRSaNHEDIs8/b0X8jDEZKyPHKELXubPrburSxZXhOPLIqCMyJiXt3LmT3Nxctm/fHnUoWaNixYrUrFmTcglcKtkSRVA7drg1qsuUgauvhquugosvtvpMxsSRm5tLlSpVqFOnDmL/K6FTVTZu3Ehubi5169ZN2H7TNlEMGeLORC1o3jx31lNCTZ8OffvCtdfCTTdBt24JfgFjMtP27dstSSSRiFC9enU2bNiQ0P2m7RiFf76EX0LnTvzyC9x6K5x6Kvz8M9Svn6AdG5M9LEkkVxjHO21bFBDyfImPPnJF/Fatguuvh4cegqpVQ3oxY4xJXWnbogjdrl1uTGLKFBg82JKEMWlszJgxiAhffPHF3vsmT55M586d99muT58+jB49GnAD8QMHDqR+/fo0b96cU045hfHjx5c6loceeoh69epx3HHHMSFvDlYBrVu3pmnTpjRt2pQjjzySCy+8EHBjEDfffDP16tWjSZMmfPbZZ6WOJ4i0blEk3JgxrojfXXe5In6LFll9JmMywMiRIzn99NMZOXIk999/f6Dn/PnPf2bdunUsXLiQChUq8O233zJlypRSxbF48WJGjRrFokWL+OabbzjrrLNYtmwZZQvMvfroo4/2Xu/atStdunQBYPz48Sxfvpzly5czY8YMrrvuOmbMmFGqmIKwT0GAb791g9Svvw7Nm7vFhcqXtyRhTALdckviC3U2bQr//Gf8bbZs2cK0adOYNGkS559/fqBEsXXrVoYOHcqqVauoUKECAIcddhjdu3cvVbxvvfUWPXr0oEKFCtStW5d69eoxc+ZMTjnllEK3/+mnn5g4cSL/+c9/9j6/d+/eiAgnn3wymzdvZt26dRxxxBGliqso2d31pAovvggNG8Jbb8Hf/ubOcLIifsZkjLfeeouOHTty7LHHUr16debMmVPkc1asWEHt2rWpGqDL+dZbb93bTeS/PPzww7/Zdu3atdSqVWvv7Zo1a7J27dqY+x4zZgzt27ffG0dxn58o2f2VefVqNyciJ8fNrj7++KgjMiZjFfXNPywjR45kwIABAPTo0YORI0fSokWLmGcHFfesoSeeeKLUMcYycuRIrr766tD2H1T2JYq8In7nnuuK+H38sav2avWZjMk4P/zwAxMnTmTBggWICLt370ZEePTRR6levTqbNm36zfaHHHII9erVY/Xq1fz0009FtipuvfVWJk2a9Jv7e/TowcCBA/e5r0aNGqxZs2bv7dzcXGrUqFHofr///ntmzpzJm2++WaLnJ5SqptWlRYsWqqratq27FMvSpaqtW6uC6uTJxXyyMaa4Fi9eHOnrP/fcc9q/f/997mvTpo1OmTJFt2/frnXq1Nkb41dffaW1a9fWzZs3q6rqnXfeqX369NEdO3aoqup3332nr732WqniWbhwoTZp0kS3b9+uK1eu1Lp16+quXbsK3faZZ57R3r1773PfO++8ox07dtQ9e/bop59+qieddFKhzy3suAOztYSfu9kxRrFrFzzyiCvit2AB/Oc/0KZN1FEZY0I2cuRILrroon3u69q1KyNHjqRChQq89NJLXHnllTRt2pRu3boxbNgwqlWrBsADDzzAoYceSsOGDWncuDGdO3cONGYRT6NGjejevTsNGzakY8eODB48eO8ZT506deKbb77Zu+2oUaPo2bPnPs/v1KkTRx99NPXq1aNfv348/fTTpYonKHGJJn3k5OTo7NmzadfO3Q404e6cc+D99+H3v3dzIg4/PMQIjTF5lixZQoMGDaIOI+sUdtxFZI6q5pRkf5k7RrF9u5swV7Ys9O/vLl27Rh2VMcaknczsevr4Y3eC9eDB7nbXrpYkjDGmhNIuUSxdCu3axZi4s2UL3HyzW0Ro+3awJq8xkUu37u10F8bxTrtEsW2b+/mbKrFTpkDjxvDvf8ONN8LChdChQyQxGmOcihUrsnHjRksWSaLeehQVK1ZM6H7TboyiUqU4A9j77++qvp52WjJDMsbEULNmTXJzcxO+PoKJLW+Fu0RKu7OeqlTJ0Z9/nu1u/Pe/8MUXcPfd7vbu3TZxzhhjClGas55C7XoSkY4islREVojIwEIeryAir3qPzxCROoF2vH69W2Wua1d480349Vd3vyUJY4xJuNAShYiUBQYD5wINgZ4i0rDAZn2BTapaD3gCeKSo/VbbudENUr/zjltM6JNPrIifMcaEKMwWRUtghaquVNVfgVFAlwLbdAFe8K6PBtpLERW5DtvxtRu0/vxzGDjQzZUwxhgTmjAHs2sAa3y3c4FWsbZR1V0i8iNQHfjev5GI9Af6ezd3yLRpC63SKwCHUOBYZTE7FvnsWOSzY5HvuJI+MS3OelLVIcAQABGZXdIBmUxjxyKfHYt8dizy2bHIJyKzS/rcMLue1gK1fLdrevcVuo2I7AdUAzaGGJMxxphiCjNRzALqi0hdESkP9ADGFthmLHCFd70bMFHT7XxdY4zJcKF1PXljDjcCE4CywHBVXSQig3B10ccCzwMvisgK4AdcMinKkLBiTkN2LPLZschnxyKfHYt8JT4WaTfhzhhjTHKlXa0nY4wxyWWJwhhjTFwpmyhCK/+RhgIci9tEZLGIzBeRD0XkqCjiTIaijoVvu64ioiKSsadGBjkWItLd+9tYJCKvJDvGZAnwP1JbRCaJyFzv/6RTFHGGTUSGi8h3IrIwxuMiIk95x2m+iDQPtOOSLrYd5gU3+P0lcDRQHvgcaFhgm+uBZ73rPYBXo447wmNxBrC/d/26bD4W3nZVgKnAdCAn6rgj/LuoD8wFDvJu/y7quCM8FkOA67zrDYGvoo47pGPRBmgOLIzxeCdgPCDAycCMIPtN1RZFKOU/0lSRx0JVJ6nqVu/mdNyclUwU5O8C4K+4umHbkxlckgU5Fv2Awaq6CUBVv0tyjMkS5FgoUNW7Xg34JonxJY2qTsWdQRpLF+D/1JkOHCgiRxS131RNFIWV/6gRaxtV3QXklf/INEGOhV9f3DeGTFTksfCa0rVU9d1kBhaBIH8XxwLHisjHIjJdRDomLbrkCnIs7gN6iUguMA64KTmhpZzifp4AaVLCwwQjIr2AHKBt1LFEQUTKAI8DfSIOJVXsh+t+aodrZU4VkRNUdXOkUUWjJzBCVf8hIqfg5m81VtU9UQeWDlK1RWHlP/IFORaIyFnAPcAFqrojSbElW1HHogrQGJgsIl/h+mDHZuiAdpC/i1xgrKruVNVVwDJc4sg0QY5FX+A1AFX9FKiIKxiYbQJ9nhSUqonCyn/kK/JYiEgz4DlcksjUfmgo4lio6o+qeoiq1lHVOrjxmgtUtcTF0FJYkP+RMbjWBCJyCK4ramUyg0ySIMdiNdAeQEQa4BJFNq7POhbo7Z39dDLwo6quK+pJKdn1pOGV/0g7AY/Fo8ABwOveeP5qVb0gsqBDEvBYZIWAx2ICcLaILAZ2A3eqasa1ugMei9uBoSJyK25gu08mfrEUkZG4LweHeOMx9wLlAFT1Wdz4TCdgBbAVuDLQfjPwWBljjEmgVO16MsYYkyIsURhjjInLEoUxxpi4LFEYY4yJyxKFMcaYuCxRmJQkIrtFZJ7vUifOtlsS8HojRGSV91qfebN3i7uPYSLS0Lt+d4HHPiltjN5+8o7LQhF5W0QOLGL7pplaKdUkj50ea1KSiGxR1QMSvW2cfYwA3lHV0SJyNvCYqjYpxf5KHVNR+xWRF4Blqvq3ONv3wVXQvTHRsZjsYS0KkxZE5ABvrY3PRGSBiPymaqyIHCEiU33fuFt7958tIp96z31dRIr6AJ8K1POee5u3r4Uicot3X2UReVdEPvfuv8S7f7KI5IjIw0AlL46Xvce2eD9Hich5vphHiEg3ESkrIo+KyCxvnYBrAhyWT/EKuolIS+89zhWRT0TkOG+W8iDgEi+WS7zYh4vITG/bwqrvGrOvqOun28UuhV1wM4nneZc3cVUEqnqPHYKbWZrXIt7i/bwduMe7XhZX++kQ3Ad/Ze/+PwJ/KeT1RgDdvOsXAzOAFsACoDJu5vsioBnQFRjqe2417+dkvPUv8mLybZMX40XAC9718rhKnpWA/sCfvPsrALOBuoXEucX3/l4HOnq3qwL7edfPAt7wrvcB/u17/oNAL+/6gbj6T5Wj/n3bJbUvKVnCwxhgm6o2zbshIuWAB0WkDbAH9036MGC97zmzgOHetmNUdZ6ItMUtVPOxV96kPO6beGEeFZE/4WoA9cXVBnpTVX/xYvgv0Bp4D/iHiDyC6676qBjvazzwpIhUADoCU1V1m9fd1UREunnbVcMV8FtV4PmVRGSe9/6XAP/zbf+CiNTHlagoF+P1zwYuEJE7vNsVgdrevowplCUKky4uAw4FWqjqTnHVYSv6N1DVqV4iOQ8YISKPA5uA/6lqzwCvcaeqjs67ISLtC9tIVZeJW/eiE/CAiHyoqoOCvAlV3S4ik4FzgEtwi+yAW3HsJlWdUMQutqlqUxHZH1fb6AbgKdxiTZNU9SJv4H9yjOcL0FVVlwaJ1xiwMQqTPqoB33lJ4gzgN+uCi1sr/FtVHQoMwy0JOR04TUTyxhwqi8ixAV/zI+BCEdlfRCrjuo0+EpEjga2q+hKuIGNh6w7v9Fo2hXkVV4wtr3UC7kP/urzniMix3msWSt2KhjcDt0t+mf28ctF9fJv+jOuCyzMBuEm85pW4ysPGxGWJwqSLl4EcEVkA9Aa+KGSbdsDnIjIX9239SVXdgPvgHCki83HdTscHeUFV/Qw3djETN2YxTFXnAicAM70uoHuBBwp5+hBgft5gdgHv4xaX+kDd0p3gEtti4DMRWYgrGx+3xe/FMh+3KM/fgYe89+5/3iSgYd5gNq7lUc6LbZF325i47PRYY4wxcVmLwhhjTFyWKIwxxsRlicIYY0xcliiMMcbEZYnCGGNMXJYojDHGxGWJwhhjTFz/D1SzEREX+2mUAAAAAElFTkSuQmCC\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, val_dataloader)\n",
        "\n",
        "# Evaluate the Bert classifier\n",
        "evaluate_roc(probs, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UNR9pBOAGz8p",
        "outputId": "97c24e9c-27c5-4917-b4fc-1204c36845b4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  122\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.29      0.92      0.45       116\n",
            "           1       0.93      0.31      0.46       369\n",
            "\n",
            "    accuracy                           0.45       485\n",
            "   macro avg       0.61      0.61      0.45       485\n",
            "weighted avg       0.78      0.45      0.46       485\n",
            "\n",
            "0.4536082474226804\n",
            "0.4602851323828921\n"
          ]
        }
      ],
      "source": [
        "PATH = './Model/BERT_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_val, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7eKaw3r2bAwg",
        "outputId": "e84554cf-46e2-4590-a4fe-84043acc2cc0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  795\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.10      0.94      0.18       475\n",
            "           1       0.96      0.16      0.28      4752\n",
            "\n",
            "    accuracy                           0.23      5227\n",
            "   macro avg       0.53      0.55      0.23      5227\n",
            "weighted avg       0.89      0.23      0.27      5227\n",
            "\n",
            "0.23225559594413622\n",
            "0.27654588065621055\n"
          ]
        }
      ],
      "source": [
        "rhi_data = pd.read_csv('data/_RHI_text.csv')\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv')\n",
        "X_test = rhi_data.text.values\n",
        "y_test = rhi_y.isRumor.values\n",
        "\n",
        "PATH = './Model/BERT_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_test, y_test)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CSx6mKV2bgB_"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2_qMkhDfb-6"
      },
      "source": [
        "# BERTweet"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "yMD4JSaapG-T"
      },
      "source": [
        "## Functions and Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 49,
      "metadata": {
        "id": "EbsgwY9ZfhWq"
      },
      "outputs": [],
      "source": [
        "def text_preprocessing(text): # Create a function to tokenize a set of texts\n",
        "\n",
        "    \"\"\"\n",
        "    - Remove entity mentions (eg. '@united')\n",
        "    - Correct errors (eg. '&amp;' to '&')\n",
        "    @param    text (str): a string to be processed.\n",
        "    @return   text (Str): the processed string.\n",
        "    \"\"\"\n",
        "    # Remove '@name'\n",
        "    text = text.lower()\n",
        "    # text = re.sub(r'(@.*?)[\\s]', ' ', text)\n",
        "    # text = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', text)\n",
        "\n",
        "    # text = re.sub(r\"http\\S+\", \"*\", text)  # http link -> '*'\n",
        "\n",
        "    # text = re.sub(r\"@\\S+\", \"@\", text)   # mention -> '@'\n",
        "    # text = re.sub(r\"@[^\\s]+\", \"@\", text)   # mention -> '@'\n",
        "\n",
        "    # sent = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', sent)\n",
        "    # sent = re.sub(r'([^\\s\\w@#\\*]|_)+', '', sent) # Erasing Special Characters\n",
        "\n",
        "    # Replace '&amp;' with '&'\n",
        "    text = re.sub(r'&amp;', '&', text)\n",
        "\n",
        "    # Remove trailing whitespace\n",
        "    text = re.sub(r'\\s+', ' ', text).strip()\n",
        "\n",
        "    return text\n",
        "\n",
        "def initialize_model(epochs=4, freeze_bert=False):\n",
        "    \"\"\"Initialize the Bert Classifier, the optimizer and the learning rate scheduler.\n",
        "    \"\"\"\n",
        "    # Instantiate Bert Classifier\n",
        "    bert_classifier = BertTweetClassifier(freeze_bert=False)\n",
        "\n",
        "    # Tell PyTorch to run the model on GPU\n",
        "    bert_classifier.to(device)\n",
        "\n",
        "    # Create the optimizer\n",
        "    optimizer = AdamW(bert_classifier.parameters(),\n",
        "                      lr=5e-5,    # Default learning rate\n",
        "                      eps=1e-8    # Default epsilon value\n",
        "                      )\n",
        "\n",
        "    # Total number of training steps\n",
        "    total_steps = len(train_dataloader) * epochs\n",
        "\n",
        "    criterion = nn.CrossEntropyLoss()\n",
        "\n",
        "    # Set up the learning rate scheduler\n",
        "    scheduler = get_linear_schedule_with_warmup(optimizer,\n",
        "                                                num_warmup_steps=0,  # Default value\n",
        "                                                num_training_steps=total_steps)\n",
        "    return bert_classifier, optimizer, scheduler, criterion\n",
        "\n",
        "class BertTweetClassifier(nn.Module): # Create the BertClassfier class\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertTweetClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 50, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = AutoModel.from_pretrained(\"vinai/bertweet-base\")\n",
        "\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            # nn.Dropout(0.5),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "\n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits\n",
        "\n",
        "    def embedding(self, input_ids):\n",
        "        outputs = self.bert(input_ids=input_ids)\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        return last_hidden_state_cls  # sequence_output, pooled_output, (hidden_states), (attentions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "1OEMCswD_0Uk"
      },
      "source": [
        "### Model Summary"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M181HsetpgSU"
      },
      "source": [
        "## Execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 44,
      "metadata": {
        "id": "v334DoYDfhWs"
      },
      "outputs": [],
      "source": [
        "raw_text = fetchdata('pheme','text')\n",
        "y = pd.read_csv('./data/_PHEME_target.csv')\n",
        "data = pd.concat([raw_text.text, y], axis=1).reset_index(drop=True)\n",
        "val = pd.read_csv('data/_PHEMEext_text.csv')\n",
        "\n",
        "X_train = data.text.values\n",
        "y_train = data.target.values\n",
        "\n",
        "X_val = val.drop(['Event'],axis=1).text.values\n",
        "y_val = val.target.values\n",
        "\n",
        "rhi_data = pd.read_csv('data/_RHI_text.csv').text.values\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv').isRumor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 45,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OwRCb2rGsFQ8",
        "outputId": "8eea1dfc-b47f-4aec-f175-a21b43771955"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(5802,) (5802,)\n",
            "(485,) (485,)\n"
          ]
        }
      ],
      "source": [
        "print(X_train.shape, y_train.shape)\n",
        "print(X_val.shape, y_val.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 46,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4ZUhevayfhWu",
        "outputId": "a9aab499-074f-4158-f61e-af51739a8c3e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla T4\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Max length:  58\n",
            "\n",
            "Tokenizing data...\n"
          ]
        }
      ],
      "source": [
        "device = getDevice()\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
        "train_dataloader, val_dataloader = data_process(X_train, y_train, X_val, y_val, batch_size=32)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 51,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vtfc0vdYfhWu",
        "outputId": "ea7ad875-ea2f-45bc-d3c3-1c0838f792a0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Start training...\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   1    |   20    |   0.659573   |     -      |     -     |   4.53   \n",
            "   1    |   40    |   0.629747   |     -      |     -     |   4.23   \n",
            "   1    |   60    |   0.657013   |     -      |     -     |   4.30   \n",
            "   1    |   80    |   0.686459   |     -      |     -     |   4.27   \n",
            "   1    |   100   |   0.658842   |     -      |     -     |   4.32   \n",
            "   1    |   120   |   0.555966   |     -      |     -     |   4.39   \n",
            "   1    |   140   |   0.659985   |     -      |     -     |   4.39   \n",
            "   1    |   160   |   0.597762   |     -      |     -     |   4.41   \n",
            "   1    |   180   |   0.613750   |     -      |     -     |   4.42   \n",
            "   1    |   200   |   0.628426   |     -      |     -     |   4.39   \n",
            "   1    |   220   |   0.556241   |     -      |     -     |   4.38   \n",
            "   1    |   240   |   0.597420   |     -      |     -     |   4.34   \n",
            "   1    |   260   |   0.432295   |     -      |     -     |   4.29   \n",
            "   1    |   280   |   0.627198   |     -      |     -     |   4.30   \n",
            "   1    |   300   |   0.616726   |     -      |     -     |   4.30   \n",
            "   1    |   320   |   0.543322   |     -      |     -     |   4.28   \n",
            "   1    |   340   |   0.493082   |     -      |     -     |   4.28   \n",
            "   1    |   360   |   0.582916   |     -      |     -     |   4.28   \n",
            "   1    |   380   |   0.582180   |     -      |     -     |   4.27   \n",
            "   1    |   400   |   0.484188   |     -      |     -     |   4.29   \n",
            "   1    |   420   |   0.486700   |     -      |     -     |   4.28   \n",
            "   1    |   440   |   0.509040   |     -      |     -     |   4.27   \n",
            "   1    |   460   |   0.485170   |     -      |     -     |   4.28   \n",
            "   1    |   480   |   0.515634   |     -      |     -     |   4.33   \n",
            "   1    |   500   |   0.537844   |     -      |     -     |   4.30   \n",
            "   1    |   520   |   0.457303   |     -      |     -     |   4.31   \n",
            "   1    |   540   |   0.554752   |     -      |     -     |   4.35   \n",
            "   1    |   560   |   0.530251   |     -      |     -     |   4.34   \n",
            "   1    |   580   |   0.624083   |     -      |     -     |   4.34   \n",
            "   1    |   600   |   0.491079   |     -      |     -     |   4.33   \n",
            "   1    |   620   |   0.426749   |     -      |     -     |   4.33   \n",
            "   1    |   640   |   0.483113   |     -      |     -     |   4.29   \n",
            "   1    |   660   |   0.435798   |     -      |     -     |   4.30   \n",
            "   1    |   680   |   0.620480   |     -      |     -     |   4.31   \n",
            "   1    |   700   |   0.548622   |     -      |     -     |   4.31   \n",
            "   1    |   720   |   0.408123   |     -      |     -     |   4.29   \n",
            "   1    |   725   |   0.528191   |     -      |     -     |   0.97   \n",
            "----------------------------------------------------------------------\n",
            "   1    |    -    |   0.554900   |  1.364816  |   35.00   |  160.21  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   2    |   20    |   0.413140   |     -      |     -     |   4.55   \n",
            "   2    |   40    |   0.400281   |     -      |     -     |   4.30   \n",
            "   2    |   60    |   0.472076   |     -      |     -     |   4.32   \n",
            "   2    |   80    |   0.436880   |     -      |     -     |   4.31   \n",
            "   2    |   100   |   0.426633   |     -      |     -     |   4.30   \n",
            "   2    |   120   |   0.430329   |     -      |     -     |   4.33   \n",
            "   2    |   140   |   0.563832   |     -      |     -     |   4.33   \n",
            "   2    |   160   |   0.523435   |     -      |     -     |   4.32   \n",
            "   2    |   180   |   0.425172   |     -      |     -     |   4.31   \n",
            "   2    |   200   |   0.512915   |     -      |     -     |   4.33   \n",
            "   2    |   220   |   0.508235   |     -      |     -     |   4.32   \n",
            "   2    |   240   |   0.398439   |     -      |     -     |   4.32   \n",
            "   2    |   260   |   0.563352   |     -      |     -     |   4.33   \n",
            "   2    |   280   |   0.450344   |     -      |     -     |   4.35   \n",
            "   2    |   300   |   0.379644   |     -      |     -     |   4.31   \n",
            "   2    |   320   |   0.499291   |     -      |     -     |   4.35   \n",
            "   2    |   340   |   0.488153   |     -      |     -     |   4.32   \n",
            "   2    |   360   |   0.468952   |     -      |     -     |   4.31   \n",
            "   2    |   380   |   0.407926   |     -      |     -     |   4.32   \n",
            "   2    |   400   |   0.486794   |     -      |     -     |   4.29   \n",
            "   2    |   420   |   0.503911   |     -      |     -     |   4.29   \n",
            "   2    |   440   |   0.494676   |     -      |     -     |   4.29   \n",
            "   2    |   460   |   0.581766   |     -      |     -     |   4.27   \n",
            "   2    |   480   |   0.549167   |     -      |     -     |   4.32   \n",
            "   2    |   500   |   0.503948   |     -      |     -     |   4.31   \n",
            "   2    |   520   |   0.474535   |     -      |     -     |   4.29   \n",
            "   2    |   540   |   0.375885   |     -      |     -     |   4.27   \n",
            "   2    |   560   |   0.454031   |     -      |     -     |   4.30   \n",
            "   2    |   580   |   0.425692   |     -      |     -     |   4.31   \n",
            "   2    |   600   |   0.434195   |     -      |     -     |   4.31   \n",
            "   2    |   620   |   0.437069   |     -      |     -     |   4.31   \n",
            "   2    |   640   |   0.451401   |     -      |     -     |   4.30   \n",
            "   2    |   660   |   0.442450   |     -      |     -     |   4.32   \n",
            "   2    |   680   |   0.469859   |     -      |     -     |   4.31   \n",
            "   2    |   700   |   0.386875   |     -      |     -     |   4.31   \n",
            "   2    |   720   |   0.411033   |     -      |     -     |   4.31   \n",
            "   2    |   725   |   0.272639   |     -      |     -     |   0.95   \n",
            "----------------------------------------------------------------------\n",
            "   2    |    -    |   0.461188   |  1.802494  |   28.44   |  160.00  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Counter 1 of 5\n",
            " Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n",
            "----------------------------------------------------------------------\n",
            "   3    |   20    |   0.466843   |     -      |     -     |   4.51   \n",
            "   3    |   40    |   0.413714   |     -      |     -     |   4.30   \n",
            "   3    |   60    |   0.509632   |     -      |     -     |   4.32   \n",
            "   3    |   80    |   0.462916   |     -      |     -     |   4.30   \n",
            "   3    |   100   |   0.330698   |     -      |     -     |   4.32   \n",
            "   3    |   120   |   0.392487   |     -      |     -     |   4.29   \n",
            "   3    |   140   |   0.383707   |     -      |     -     |   4.29   \n",
            "   3    |   160   |   0.402919   |     -      |     -     |   4.31   \n",
            "   3    |   180   |   0.501942   |     -      |     -     |   4.31   \n",
            "   3    |   200   |   0.396362   |     -      |     -     |   4.30   \n",
            "   3    |   220   |   0.354979   |     -      |     -     |   4.29   \n",
            "   3    |   240   |   0.369049   |     -      |     -     |   4.31   \n",
            "   3    |   260   |   0.400176   |     -      |     -     |   4.30   \n",
            "   3    |   280   |   0.360281   |     -      |     -     |   4.28   \n",
            "   3    |   300   |   0.425081   |     -      |     -     |   4.27   \n",
            "   3    |   320   |   0.367919   |     -      |     -     |   4.30   \n",
            "   3    |   340   |   0.457043   |     -      |     -     |   4.28   \n",
            "   3    |   360   |   0.436701   |     -      |     -     |   4.28   \n",
            "   3    |   380   |   0.350623   |     -      |     -     |   4.28   \n",
            "   3    |   400   |   0.445031   |     -      |     -     |   4.30   \n",
            "   3    |   420   |   0.454515   |     -      |     -     |   4.29   \n",
            "   3    |   440   |   0.452058   |     -      |     -     |   4.28   \n",
            "   3    |   460   |   0.390991   |     -      |     -     |   4.30   \n",
            "   3    |   480   |   0.439889   |     -      |     -     |   4.28   \n",
            "   3    |   500   |   0.524050   |     -      |     -     |   4.31   \n",
            "   3    |   520   |   0.338033   |     -      |     -     |   4.29   \n",
            "   3    |   540   |   0.388898   |     -      |     -     |   4.30   \n",
            "   3    |   560   |   0.414471   |     -      |     -     |   4.29   \n",
            "   3    |   580   |   0.422597   |     -      |     -     |   4.30   \n",
            "   3    |   600   |   0.383847   |     -      |     -     |   4.28   \n",
            "   3    |   620   |   0.285526   |     -      |     -     |   4.28   \n",
            "   3    |   640   |   0.504382   |     -      |     -     |   4.30   \n",
            "   3    |   660   |   0.336051   |     -      |     -     |   4.29   \n",
            "   3    |   680   |   0.426840   |     -      |     -     |   4.28   \n",
            "   3    |   700   |   0.459893   |     -      |     -     |   4.28   \n",
            "   3    |   720   |   0.353509   |     -      |     -     |   4.27   \n",
            "   3    |   725   |   0.322742   |     -      |     -     |   0.95   \n",
            "----------------------------------------------------------------------\n",
            "   3    |    -    |   0.410680   |  1.668548  |   35.00   |  159.33  \n",
            "----------------------------------------------------------------------\n",
            "\n",
            "\n",
            "Counter 2 of 5\n",
            "Early stopping with best_acc:  35.0 and val_acc for this epoch:  35.0 ...\n",
            "Training complete!\n"
          ]
        }
      ],
      "source": [
        "set_seed(42)    # Set seed for reproducibility\n",
        "bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=4, freeze_bert=True)\n",
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=4, evaluation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "xqI9LRVbfhWu"
      },
      "outputs": [],
      "source": [
        "torch.save(bert_classifier.state_dict(), './Model/BERTweet_raw_to_fine_tune_ord5.pt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 327
        },
        "id": "1E0J9rlrfhWu",
        "outputId": "25c16e02-c092-49d7-eec5-6b007e74f157"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "AUC: 0.5061\n",
            "Accuracy: 34.85%\n"
          ]
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5gUVdbH8e8BSSqiYiYIqxgQEXEEMYEZEUUXVjGjIKirYlwxx1V5MazuYsCwhlUMuCJGXBXELCCgBEUEgQFRRAyIIOB5/7g1TjHM9PTMdE33zPw+z9PPdHdVV52umenTt27dc83dERERKUmtbAcgIiK5TYlCRERSUqIQEZGUlChERCQlJQoREUlJiUJERFJSopAyMbNpZtYl23HkCjO73MweyNK+HzazG7Ox70wzsxPN7LVyvlZ/kwlToqjCzOwrM/vVzJaZ2aLog2PDJPfp7ru4+9gk91HAzOqZ2c1mNi96n1+Y2SVmZpWx/2Li6WJm+fHn3P0md++X0P7MzM4zs6lm9ouZ5ZvZM2a2axL7Ky8zu9bM/lORbbj74+5+aBr7Wic5VubfZE2lRFH1HenuGwLtgN2By7IcT5mZ2XolLHoGOAjoBjQETgb6A3cmEIOZWa79P9wJDATOAzYFdgBGAkdkekcpfgeJy+a+JU3urlsVvQFfAQfHHv8f8FLs8V7Ae8APwBSgS2zZpsC/gYXAUmBkbFl3YHL0uveAtkX3CWwD/ApsGlu2O/AdUCd6fDowI9r+aGDb2LoO/BX4AphTzHs7CFgBNCvyfEdgDbB99HgscDPwEfAT8HyRmFIdg7HA34F3o/eyPXBaFPPPwGxgQLTuBtE6vwPLots2wLXAf6J1WkTv61RgXnQsrojtrwHwSHQ8ZgB/A/JL+N22it5nhxS//4eBocBLUbwfAtvFlt8JzI+Oy0Rgv9iya4ERwH+i5f2ADsD70bH6GvgXUDf2ml2A/wHfA98AlwNdgd+AVdExmRKt2wh4MNrOAuBGoHa0rE90zO8AlkTL+gDvRMstWvZtFNunQBvCl4RV0f6WAS8U/T8AakdxfRkdk4kU+RvSrRyfNdkOQLcK/PLW/gdpGv1D3Rk9bhL9E3YjtBwPiR5vHi1/CXgK2ASoA3SOnt89+gftGP3TnRrtp14x+3wTOCMWzxDg3uh+D2AWsDOwHnAl8F5sXY8+dDYFGhTz3m4B3irhfc+l8AN8bPRB1IbwYf4shR/cpR2DsYQP9F2iGOsQvq1vF31YdQaWA+2j9btQ5IOd4hPF/YSksBuwEtg5/p6iY94U+KTo9mLbPROYW8rv/+Ho/XSI4n8ceDK2/CSgcbTsImARUD8W9yrg6OjYNAD2ICTW9aL3MgM4P1q/IeFD/yKgfvS4Y9FjENv3c8B90e9kC0IiL/id9QFWA+dG+2rA2oniMMIH/MbR72FnYOvYe74xxf/BJYT/gx2j1+4GNM72/2pVv2U9AN0q8MsL/yDLCN+cHHgD2DhadinwWJH1RxM++LcmfDPepJht3gPcUOS5zylMJPF/yn7Am9F9I3x73T96/ArQN7aNWoQP3W2jxw4cmOK9PRD/0Cuy7AOib+qED/tbYstaE75x1k51DGKvvb6UYzwSGBjd70J6iaJpbPlHQO/o/mzgsNiyfkW3F1t2BfBBKbE9DDwQe9wN+CzF+kuB3WJxjytl++cDz0X3jwcmlbDeH8cgerwlIUE2iD13PDAmut8HmFdkG30oTBQHAjMJSatWMe85VaL4HOiRxP9bTb7l2jlZKbuj3b0h4UNsJ2Cz6Pltgb+Y2Q8FN2BfQpJoBnzv7kuL2d62wEVFXteMcJqlqGeBTma2NbA/Ifm8HdvOnbFtfE9IJk1ir5+f4n19F8VanK2j5cVtZy6hZbAZqY9BsTGY2eFm9oGZfR+t343CY5quRbH7y4GCCwy2KbK/VO9/CSW//3T2hZldbGYzzOzH6L00Yu33UvS972BmL0YXRvwE3BRbvxnhdE46tiX8Dr6OHff7CC2LYvcd5+5vEk57DQW+NbNhZrZRmvsuS5ySJiWKasLd3yJ827o1emo+4dv0xrHbBu5+S7RsUzPbuJhNzQf+XuR167v78GL2uRR4DTgOOIHQAvDYdgYU2U4Dd38vvokUb+l1oKOZNYs/aWYdCR8Gb8aejq/TnHBK5btSjsE6MZhZPULyuxXY0t03Bl4mJLjS4k3H14RTTsXFXdQbQFMzyyvPjsxsP0IfyLGEluPGwI8UvhdY9/3cA3wGtHL3jQjn+gvWnw/8qYTdFd3OfEKLYrPYcd/I3XdJ8Zq1N+h+l7vvQWgh7kA4pVTq66J9b1fKOlJGShTVyz+AQ8xsN0In5ZFmdpiZ1Taz+tHlnU3d/WvCqaG7zWwTM6tjZvtH27gfONPMOkZXAm1gZkeYWcMS9vkEcArQK7pf4F7gMjPbBcDMGpnZX9J9I+7+OuHD8lkz2yV6D3tF7+sed/8itvpJZtbazNYHrgdGuPuaVMeghN3WBeoBi4HVZnY4EL9k8xugsZk1Svd9FPE04ZhsYmZNgHNKWjF6f3cDw6OY60bx9zazQWnsqyGhH2AxsJ6ZXQ2U9q28IaHzeJmZ7QScFVv2IrC1mZ0fXbbcMEraEI5Li4KrxqK/r9eA28xsIzOrZWbbmVnnNOLGzPaM/v7qAL8QLmr4PbavkhIWhFOWN5hZq+jvt62ZNU5nv1IyJYpqxN0XA48CV7v7fEKH8uWED4v5hG9lBb/zkwnfvD8jdF6fH21jAnAGoem/lNAh3SfFbkcRrtBZ5O5TYrE8BwwGnoxOY0wFDi/jW+oJjAFeJfTF/IdwJc25RdZ7jNCaWkToaD0viqG0Y7AWd/85eu3ThPd+QvT+CpZ/BgwHZkenVIo7HZfK9UA+MIfQYhpB+OZdkvMoPAXzA+GUyjHAC2nsazThuM0knI5bQepTXQAXE97zz4QvDE8VLIiOzSHAkYTj/AVwQLT4mejnEjP7OLp/CiHxTiccyxGkdyoNQkK7P3rdXMJpuCHRsgeB1tHxH1nMa28n/P5eIyS9Bwmd5VIBVnimQKTqMbOxhI7UrIyOrggzO4vQ0Z3WN22RbFGLQqSSmNnWZrZPdCpmR8Klps9lOy6R0iSWKMzsITP71symlrDczOwuM5tlZp+YWfukYhHJEXUJV//8TOiMf57QDyGS0xI79RR1ji4DHnX3NsUs70Y419yNMLjrTnfvWHQ9ERHJrsRaFO4+jnDtfEl6EJKIu/sHwMbR9fgiIpJDslmMqwlrX4WRHz33ddEVzaw/oc4LG2ywwR477bRTpQQoIlLlzZ3L6iU/MMVXf+fum5dnE1WiaqO7DwOGAeTl5fmECROyHJGISA4r6FIwg3vu4d+Dv+X0udfOLe/msnnV0wLWHpnaNHpORETKa8EC6NEDnojGv551Fo+0uKZCm8xmohgFnBJd/bQX8GM0olNERMrKnXEn38+ybVuz4qXXue26ZXTpAl26wOTJFdt0YqeezGw4oVDdZhZmBbuGUCgMd7+XUEOnG2Hk73LCPAAiIlJWX34JZ5zB/mPGMK72Adybdz8LGxSWvGrXDt56q/ybTyxRuPvxpSwvmLhGREQq4tNPYeJEhuwwjJe26sfYt9adLbgiEwhXic5sEZGabtiwwm4HgJa/TKXVzx/z2lanAEezUZvZjJvWmHYJDDJQCQ8RkSrgiSdCX8N6v/9Gn6+uZdjE9vT76grq/r4CgJ/qNKZdOzjhhMzvWy0KEZEcVtCSmDwZjv/Th9zzfV+YOw1OOokt7riD1zarn3gMShQiIjmsIEkcvPMC/jV+P9h6S3jxRTjiiEqLQYlCRCSHNV0+E9rtwIixTeC5p+Cgg2CjdGeGzQz1UYiI5Jhhw6D7vj/wwjb9eXT8TrT9YVxYcMwxlZ4kQC0KEZGc89Vdoxg2/Sy29EU82ewSduu3Z1bjUaIQEckl/fpx07QH+XKDXak99nlOyMvLdkRKFCIi2fLH2IhYEb+jFubxa71tGdP+Ul7Pq5vV+AooUYiIVKL4wLm33oKmzOfpTc/kzS16878tT2bUNmfCNsmMhygvJQoRkYQVTQ4AXfb/ndta3cc58y+l7oo1dLroGK7ol70YU9FVTyIiCSsYCwHQuTMMv/4LxnAAF35xNnX37QhTp0K/HM0SqEUhIlIp2rWDsWOjB89Ph9s/gYcegj59KlaxrxIoUYiIVILtlk2BRybDqaeGiYVmz4ZNNsl2WGnRqScRkQQMGxYmDTpk/5Uc8cFV3PdxHlx1FawIRfyqSpIAtShERBLxxBNQZ8L73O99abFyBjP3OoUdXrwd6idfxC/TlChERDIkfnXTNx8v4JPlnanTdCsY8TI7HH54doOrAJ16EhHJkCeegOUTZwCwZfsmvHnm0zBtGlThJAFqUYiIVEhBK2LDVUvpN/4iTlr1b7hhHOy3H3B0tsPLCCUKEZEKeOIJaDr+Oe5cfTYbr1rMpK6Xsfue2S3il2k69SQiUg4FVzX1e+90/rP8zzRuvRW1J37E7q/cVCU7rFNRi0JEpKzceeJxmDzFWNRiLz7apRUdnr4Y6tTJdmSJUKIQEUnTsGHwxkNzuWjmAHZacQJ0OIWLx/bPdliJU6IQEUnH77+zfMg9PDBrEOvVcv60/V9on0MVXpOkRCEiUprPP4d+/Th/1jt8tMmhdPj4Pv7WokW2o6o0ShQiIqX5/HOYNo2bd3yY0VuewtgWuV3EL9N01ZOISHEmTYJ//zvcP+oomD2b0VudmvOVXpOgRCEiErdiBVx+Oey5J1x7bWERv403zmpY2aREISJS4N13w8QRN98Mp5wSZhuqZmMiykN9FCIiAAsWwAEHQJMmMHo0HHpotiPKGWpRiEjNNn16+NmkCTz7LHz6qZJEEUoUIlIzff99mIZ0l11g3Ljw3JFHwoYbZjWsXKRTTyJS8zz7LPz1r7BkCVxxBXTokO2IcpoShYjULH36wCOPQPv28OqrofNaUlKiEJHqzz38NIO994add4aLLoL1Sv8IjM9aN3lyzcwrifZRmFlXM/vczGaZ2aBiljc3szFmNsnMPjGzbknGIyI10Jw5oXP60UfD4/794dJL00oSEJLE5Mnhfrt2cEINqe8Ul1iLwsxqA0OBQ4B8YLyZjXL36bHVrgSedvd7zKw18DLQIqmYRKQGWbMGhg6Fyy6DWrXgxBPTfmlxrYixY5MJsypIskXRAZjl7rPd/TfgSaBHkXUc2Ci63whYmGA8IlJTzJgRpiIdOBA6dw7zVvfpk/bL1YpYW5J9FE2A+bHH+UDHIutcC7xmZucCGwAHF7chM+sP9Ado3rx5xgMVkWpm1qxQyO+xx0JLohz1mWp6KyIu2+MojgcedvemQDfgMTNbJyZ3H+buee6et/nmm1d6kCJSBUycCA89FO4feWTomzjppBpZxC/TkkwUC4BmscdNo+fi+gJPA7j7+0B9YLMEYxKR6ubXX2HQIOjYEW64obCI30YbpX6dpC3JRDEeaGVmLc2sLtAbGFVknXnAQQBmtjMhUSxOMCYRqU7GjYPddoPBg0MfxKRJKuKXgMT6KNx9tZmdA4wGagMPufs0M7semODuo4CLgPvN7AJCx3Yf94ILnkVEUliwAA46CJo1g9dfD/clEYkOuHP3lwmXvMafuzp2fzqwT5IxiEg18+mnsOuuoYjfc8+Fiq8bbJDtqKq1bHdmi4ik57vv4OSToW3bwiJ+3bsrSVQClfAQkdzmDs88A+ecA0uXwjXXhI5rqTRKFCKS2049NYyHyMuDN94Ip50yJD4CO66m1nQqiRKFiOSeeBG/zp3D6abzz0+7PlO6CkZgF00KGo29NiUKEckts2fDGWeEwXKnnQZ9+1Z4k6W1HDQCOzV1ZotIblizBv7xj3Bqafz4UMgvQ+K1m+LUckiPWhQikn3Tp8Ppp8OHH8IRR8C990LTphndhVoO5adEISLZN2cOfPll+Orfu7fqM+UYJQoRyY7x48P5oDPOCK2I2bOhYcNsRyXFUB+FiFSu5cvh4othr73g5psLi/gpSeQsJQoRqTxjx4ZLXW+7LbQkVMSvStCpJxGpHPn5cMghsO228OaboUaTVAlqUYhIsqZMCT+bNoXnn4dPPqmUJDFsGHTpEm7FXRor6VOiEJFkLF4cBim0awdvvRWe69YN1l8/0d0WJIgBAwp3q/ESFaNTTyKSWe7w5JNw3nnw449w3XXQqVOiu4yPvC5IDp07h+TQv3+iu64RlChEJLNOPhkefzxUeH3wQdhll8R3Ga/ZpASReWknCjNb392XJxmMiFRRv/8eBsmZhf6HPfYILYratSstBI28Tk6picLM9gYeADYEmpvZbsAAdz876eBEpAqYNStc6nryyaEMRwaK+JVEZcGzI53O7DuAw4AlAO4+Bdg/yaBEpApYvRpuvTUU8Zs0CerWTWQ38auX4h3UceqsTlZap57cfb6tXXtlTTLhiEiVMHVqKAE+YQL06AF33w3bbJPIrtT/kH3pJIr50eknN7M6wEBgRrJhiUhOmzcP5s4NVzcde2ziRfzU/5Bd6SSKM4E7gSbAAuA1QP0TIjXNhx+GwXP9+4fxELNnw4YbJrKreF+E+h+yL50+ih3d/UR339Ldt3D3k4Cdkw5MRHLEL7/AhReGsRD/93+wcmV4PqEkAWtPNKT+h+xLp0XxT6B9Gs+JSHXz5pvhiqbZs+Gss+CWW6BevUrZtU435Y4SE4WZdQL2BjY3swtjizYCKu/iaBHJjvx8OOwwaNkyXGq0f7IXO+p0U+5K1aKoSxg7sR4QLxT/E9AryaBEJIsmTYLddw9F/F54IVxq1KBBIrsqqfSGTjflFnP31CuYbevucyspnlLl5eX5hAkTsh2GSPXzzTdhNPXTT4dzPp07V3iTJQ2QKxBPDqBLX5NkZhPdPa88r02nj2K5mQ0BdgH+mGHE3Q8szw5FJMe4h9pMAwfCsmVw442w997l3lxJrYTiaFxE1ZBOongceAroTrhU9lRgcZJBiUglOuGEMB6iU6dQxG/nil3UqAFy1U86iaKxuz9oZgPd/S3gLTMbn3RgIpKgeBG/Qw8NSeKvf81YET9dsVS9pDOOYlX082szO8LMdgc2TTAmEUnSzJmhwutDD4XHp51W6ZVepWpJJ1HcaGaNgIuAiwmVZM9PNCoRybzVq8OAud12C9ORZvBKJk07Wr2VeurJ3V+M7v4IHABgZvskGZSIZNgnn4QS4BMnwjHHwNChsPXWFdqkLm2tOVINuKsNHEuo8fSqu081s+7A5UADYPfKCVFEKiw/H+bPh2eegZ49y13Er6TkoE7r6i1Vi+JBoBnwEXCXmS0E8oBB7j4ynY2bWVdCQcHawAPufksx6xwLXAs4MMXd9V1EJBPeey+0JM48s7CI3wYblHkzSg6SKlHkAW3d/Xczqw8sArZz9yXpbDhqkQwFDgHygfFmNsrdp8fWaQVcBuzj7kvNbIvyvhERiSxbBldcAf/8J2y3XeisrlevXEkCdLmrpE4Uv7n77wDuvsLMZqebJCIdgFnuPhvAzJ4EegDTY+ucAQx196XRfr4tU/QisrbXXguf4vPmhctdb7opI0X8dLlrzZYqUexkZp9E9w3YLnpsgLt721K23QSYH3ucD3Qsss4OAGb2LuH01LXu/mrRDZlZf6A/QPPmzUvZrUgNNX8+HHFEaEWMGwf77lvmTRRXckMF+iRVoqiMOSfWA1oBXYCmwDgz29Xdf4iv5O7DgGEQaj1VQlwiVcfEibDHHtCsGbz8Muy3H9SvX/rrihE/zVRAVzFJiYkiA4UAFxA6wws0jZ6Lywc+dPdVwBwzm0lIHBr5LVKaRYvg3HNhxIjCIn6HHFLhzeo0kxSVzoC78hoPtDKzlmZWF+gNjCqyzkhCawIz24xwKmp2gjGJVH3u8Mgj0Lp1KAN+000VLuKnwXKSSmKJwt1XA+cAo4EZwNPuPs3Mrjezo6LVRgNLzGw6MAa4pIwd5iI1T+/e0KdPSBSTJ8Nll0GdOuXenKYdldKUOh8FgJk1AJq7++fJh5Sa5qOQGilexO+RR+Dnn+Hss6FW+t/1SpoboqBPQqebqreKzEdR6l+ZmR0JTAZejR63M7Oip5BEJCmffRamIX3wwfD41FPhnHPKlCRg7ZZDnFoRUpp0yoxfSxgTMRbA3SebWcsEYxIRgFWrYMgQuO66MFhuww0rvEm1HKQ80kkUq9z9R1u7NowuURVJ0uTJYUT15MnQq1cYZb3VVmXeTPx0k8ZDSHmlkyimmdkJQO2o5MZ5wHvJhiVSwy1aFG7PPgt//vMfT5c2B3VRquoqmVBqZ7aZrQ9cARwaPTUauNHdVyQcW7HUmS3V1jvvhCJ+Z58dHi9fDuuvv9YqBZewlqVloNpMAhXrzE6nRbGTu19BSBYikgHxlkGD1T/Tf85lHLNwKPMbtOL0J/uyqlY9YP11XqcrlCQb0rls4jYzm2FmN5hZm8QjEqkBCq5A2vP70fx7Qht6LLybEU0G0n+Pj6MkUTydPpJsSGeGuwPMbCvCJEb3mdlGwFPufmPi0YlUI0U7lg/ZaT5DJnaH7beHB9+h19570yu7IYoUK51TT7j7IsLkRWOAvwFXA0oUIqVYd9IfZ8Du46FdBw45oRnc+Eqo8lrOIn4ilaHURGFmOwPHAT2BJcBTwEUJxyVSLcSrsR6z19fctuKvtJz0XGERPw7OdogipUqnRfEQITkc5u4LE45HpNppt5sz9tSH4cILYcUKGDwY9tkn22GJpC2dPopOlRGISHV17fRj4fQRYZ6IBx6AHXbIdkgiZVJiojCzp939WDP7lLVHYqc7w51IjVF0IFwtX4NjTJ5Si/e2OpIu1x8IAwaUuT6TSC5I1aIYGP3sXhmBiFRl8b6I5r/M4G8z+/LKVqfh7c5gsxNOiSbyFamaUs1w93V092x3vzS+zMwGA5eu+yqRmqWgJTF5MuzRdhVvHDwYbrgBNtyQNjc04pJjsx2hSMWl0w4ubm7FwzMdiEhVEZ8RbsCAcNlrr+0m8fScPLjqKjjmGJgxA45VlpDqIVUfxVnA2cCfzOyT2KKGwLtJByaSbSUV4IsX2uvcOaql1Pwb6PsdjBwJPXpUbqAiCUvVR/EE8ApwMzAo9vzP7v59olGJZMm6A+Si4Q4xfySHncbBp59C/78CXWHWLGjQoFLjFakMqRKFu/tXZvbXogvMbFMlC6lq0inRXWxroWhH9E8/waBBMOCecKlrv35Qr56ShFRbpbUougMTCZfHxmcucuBPCcYlkhHptBDiSkwOBV5+OXRMLFwYBtBdf31IEiLVWKqrnrpHPzXtqVQ5BQkirRZCuubPD/0PO+4II0ZAx44Zi1ckl6VT62kfYLK7/2JmJwHtgX+4+7zEoxMpp4JLViucHNzhww9hr72gWTN47bVQfqNu3YzGK5LL0rk89h5guZntRigG+CXwWKJRiWRAwQQ/5U4SCxfC0UdDp06FTZMDDlCSkBonnUSx2sN8qT2Af7n7UMIlsiLVk3uoydS6dWhB3HqrivhJjZZO9difzewy4GRgPzOrBdRJNiyRLOrVC/7733De6oEHwsRCIjVYOi2K44CVwOnRBEZNgSGJRiVSDvER05Mnl/HFa9bA77+H+0cfDffeC2++qSQhQhqJIkoOjwONzKw7sMLdH008MpEyKujAhjLOLT11aji19OCD4fHJJ6vSq0hMOlc9HUtoQYwljKX4p5ld4u4jEo5NpMwKOrDT8ttvcPPN8Pe/Q6NGsMkmSYYmUmWl00dxBbCnu38LYGabA68DShSSFSWNsC4o852WiROhT5/QmjjhBPjHP2DzzTMZpki1kU7bulZBkogsSfN1IomIn2KKK9PppiVL4Icf4IUX4PHHlSREUkinRfGqmY0GhkePjwNeTi4kkaC0lkPap5gKjBkTividdx4ceih88QXUr5+JUEWqtXTmzL7EzP4M7Bs9Nczdn0s2LKmp0qnNVKaWA8CPP8Lf/hY2vtNOoaO6Xj0lCZE0pZqPohVwK7Ad8ClwsbsvqKzApGaKTyla4fIbEE4tnXkmLFoEF18M112nIn4iZZSqRfEQ8CgwDjgS+Cfw58oISmq2cp1WKs78+dCzZ2hFjBwJe+6ZgY2K1DypEkVDd78/uv+5mX1cGQGJVIg7vP8+7L13YRG/vfdWfSaRCkh19VJ9M9vdzNqbWXugQZHHpTKzrmb2uZnNMrNBKdbraWZuZnllfQNSdcVHUpd7RHVcfj4cdVQYPFfQwdGli5KESAWlalF8Ddwee7wo9tiBA1Nt2MxqA0OBQ4B8YLyZjXL36UXWawgMBD4sW+hS1cX7IwqUuaMaQumN+++HSy6B1avh9tth331Lf52IpCXVxEUHVHDbHYBZ7j4bwMyeJFSgnV5kvRuAwcAlFdyfVAHxq5rKfZlrUT17hj6IAw8MCeNPmnxRJJOSHDjXBJgfe5wfPfeH6BRWM3d/KdWGzKy/mU0wswmLFy/OfKRSacpdj6mo1asLi/j17BkSxOuvK0mIJCCdAXeJiMqV3w70KW1ddx8GDAPIy8vzZCOTpFW4FfHJJ9C3L/TrF8ZEnHRSpkITkWIk2aJYADSLPW4aPVegIdAGGGtmXwF7AaPUoS0lWrkSrrkG9tgD5s5V2Q2RSlJqorDgJDO7Onrc3Mw6pLHt8UArM2tpZnWB3sCogoXu/qO7b+buLdy9BfABcJS7TyjXO5GcVaF5IgqMHw/t28P118Pxx8OMGfBnDesRqQzptCjuBjoBx0ePfyZczZSSu68GzgFGAzOAp919mpldb2ZHlTNeqYIy0i+xdCksWwYvvwyPPgqNG2c0RhEpWTp9FB3dvb2ZTQJw96VRC6FU7v4yRQoIuvvVJazbJZ1tStVUrn6JN98MRfwGDgxF/GbOVPkNkSxIp0WxKhoT4fDHfBS/JxqVVHkVOt30ww9wxhlw0EFw332hbwKUJESyJJ0WxV3Ac8AWZvZ3oBdwZaJRSU4rqfx3XLzya5lONz3/PJx1FnzzTaj4eu21ShAiWZZOmfHHzWwicBBhKtSj3X1G4pFJzipuRHVR5ar8Om8e/OUvsPPOMGoU5OkCOJFckM6c2c2B5cAL8efcfV6SgUluy1iFV3d454sM+AMAABXOSURBVB3Ybz9o3jwMmttrL9VnEskh6fRRvAS8GP18A5gNvJJkUJJ7MnKJa1Hz5sERR8D++xeeq9p/fyUJkRxTaqJw913dvW30sxWhhtP7yYcmuSRjpTcglN64+27YZRcYNw7uuktF/ERyWJlLeLj7x2bWMYlgJLckUsAPwkC555+HQw4JO2nRIgMbFZGkpNNHcWHsYS2gPbAwsYgkZ8Q7rSvcili9GmrVCrfjjoMePaBPHzDLVLgikpB0WhQNY/dXE/oqnk0mHMk1GWlFTJkCp58exkaceWYowSEiVUbKRBENtGvo7hdXUjxSnaxYATfeCIMHw6abwlZbZTsiESmHEhOFma3n7qvNbJ/KDEiyo7hBdKWNlUjpo4/g1FPhs8/Cz9tvD8lCRKqcVC2Kjwj9EZPNbBTwDPBLwUJ3/2/CsUkCShpVHR9JXaBC/RI//QS//gqvvgqHHVbOjYhILkinj6I+sIQwR7YTRmc7oERRBZU0qrpcI6mLeu01mDYNLrgADj4YPv9c5TdEqoFUiWKL6IqnqRQmiAKaZa4Ky9hlrgWWLoULL4SHHw5jI84+OyQIJQmRaiHVgLvawIbRrWHsfsFNBP77X2jdGh57DC67DCZMUIIQqWZStSi+dvfrKy0SqXrmzYPevaFNmzCh0O67ZzsiEUlAqkShkVBVTDrlvyt0JROEIn7jxoVOjebNw+RCHTtCnToV2KiI5LJUp54OqrQopNzixfoGDCi8eqkkFbqSae5cOPzwsLOCHe27r5KESDVXYovC3b+vzECkfOJXMWXkyqXiFBTxGzQoPP7nP0NZcBGpEcpcFFCyL7FifSU5+mh44YUwHuK++2DbbRPcmYjkmnTmo5Ack9GS3yVZtSq0JCDUZnrkEXjlFSUJkRpILYoqKtFWxMcfQ9++oYjf2WeriJ9IDacWhRT69dcwFqJDB1i0CJo1y3ZEIpID1KKQ4IMPQvG+mTNDSfBbb4VNNsl2VCKSA5QoclzGq7qW5JdfQr/E//4X6jSJiER06inHxTuuC2SsA/vVV+G228L9gw4KJcGVJESkCLUoqoCMd1wvWRKK+D36KOy6K5x7LtStG24iIkWoRZGD4qOti7YmKsQdRowIRfyeeAKuvBLGj1eCEJGU1KLIQfHR1hkdJzFvXthY27Zh7ojddsvQhkWkOlOiyFEZO93kDmPGwIEHhsFyY8eGy1/X069eRNKjU0/V2Zw5cOihoaO6oIjf3nsrSYhImShRVEdr1sCdd4Z5Ij78EO65R0X8RKTc9NWyOurRA156Cbp1g3vv1QhrEakQJYrqYtUqqF0batWCk08O9ZlOOAFM80+JSMUkeurJzLqa2edmNsvMBhWz/EIzm25mn5jZG2ZWY0uTVuiS2AkTIC8vnGICOO44OPFEJQkRyYjEEoWZ1QaGAocDrYHjzax1kdUmAXnu3hYYAfxfUvHkunKVDv/1V7j00jAV6eLFKgEuIolI8tRTB2CWu88GMLMngR7A9IIV3H1MbP0PgJMSjCfnlemS2PffD0X8vvgC+vWDIUNg442TDE9EaqgkE0UTYH7scT7QMcX6fYFXiltgZv2B/gDNmzfPVHxZV9xMdWn79dcwsdDrr4fLX0VEEpITl8ea2UlAHjCkuOXuPszd89w9b/PNN6/c4BJU5tNNL78cWg4QBtDNmKEkISKJS7JFsQCIX5fZNHpuLWZ2MHAF0NndVyYYT04o13zX330H558Pjz8eym4MHBjqM9Wpk3S4IiKJJorxQCsza0lIEL2Btb4zm9nuwH1AV3f/NsFYKl1x80hA4QDpzp3TaEW4w1NPhequP/4I11wDl1+uIn4iUqkSSxTuvtrMzgFGA7WBh9x9mpldD0xw91GEU00bAs9YuJRznrsflVRMlSle2C+uc+eQHPr3T2Mj8+aFDuvddoMHHwwlwUVEKlmiA+7c/WXg5SLPXR27X61nySlXYT93eOONMIHQttuGJsiee4bBdCIiWZATndkS+fLL0Dl9yCGF56j22ktJQkSySokig8o9unrNGrj99nBqaeJEuO8+FfETkZyhWk8ZUNBxXaaO6rgjj4RXXoHu3UMZjqZNE4tVRKSslCjKIJ0rmdLuqP7ttzAvRK1a0KdPKOTXu7fqM4lIzlGiKEU8OcQTQlyZEgTARx9B374wYACccw4ce2zG4hURyTQlilLEL3Mtc0IoavlyuOoq+Mc/YOutYbvtMhqriEgSlCiKUa7R06V5550wJmL27NCSGDwYGjWqaKgiIolTooiUdIqpTJ3SqRRMLDRmTLgsSkSkilCiiGT0FFOBF14Ihfv+9jc44ACYPj10YIuIVCH61IrJyCkmCJMIDRwIw4eHjZ5/fqjPpCQhIlWQBtxlkntomuy8M4wYAddfDx9+qCJ+IlKl6StuJs2bB6edBrvvHor47bJLtiMSEakwtSgq6vffYfTocH/bbeHtt+Hdd5UkRKTaUKKoiC++CDPNde0K48aF5zp0UBE/EalWanSiKHcRv9Wrw5SkbduGFz74oIr4iUi1VWP6KIqr01Tu8RLdu4fTTT16wN13wzbbZDRWkepi1apV5Ofns2LFimyHUmPUr1+fpk2bUieDUyVX60RRWp2mMo2XWLkyzFFdqxb06wennw5/+YuK+ImkkJ+fT8OGDWnRogWm/5XEuTtLliwhPz+fli1bZmy71TpRZGwQ3QcfhCJ+Z54Z5q/u1SvjsYpURytWrFCSqERmRuPGjVm8eHFGt1utEwVUcBDdL7/AlVfCnXeGOSJatcpkaCI1gpJE5UrieFf7RFFub78divjNmQNnnw033wwbbZTtqEREKl21u+qp3FcyFbV6deiTeOstGDpUSUKkChs5ciRmxmefffbHc2PHjqV79+5rrdenTx9GjBgBhI74QYMG0apVK9q3b0+nTp145ZVXKhzLzTffzPbbb8+OO+7I6IIxWEX06dOHli1b0q5dO9q1a8fk6MPss88+o1OnTtSrV49bb721wrGkq9q1KOL9EmWu/DpyZCjid9lloYjftGmqzyRSDQwfPpx9992X4cOHc91116X1mquuuoqvv/6aqVOnUq9ePb755hveKrgqppymT5/Ok08+ybRp01i4cCEHH3wwM2fOpHYxY6+GDBlCryL9oZtuuil33XUXI0eOrFAcZVUtPwXL3C/xzTehk/qZZ6B9e7joIhXxE8mw88+vYCu/GO3ahXnAUlm2bBnvvPMOY8aM4cgjj0wrUSxfvpz777+fOXPmUK9ePQC23HJLjq3gbJTPP/88vXv3pl69erRs2ZLtt9+ejz76iE6dOqX1+i222IItttiCl156qUJxlFW1OPVU7tNN7vDYY9C6NTz/PPz97+EKJxXxE6k2nn/+ebp27coOO+xA48aNmThxYqmvmTVrFs2bN2ejNE45X3DBBX+cIorfbrnllnXWXbBgAc2aNfvjcdOmTVmwYEGx273iiito27YtF1xwAStXriw1jiRVi6/M5T7dNG9eGBORlxdGV++0U6JxitRkpX3zT8rw4cMZOHAgAL1792b48OHsscceJV4dVNarhu64444Kx1jUzTffzFZbbcVvv/1G//79GTx4MFdffXXG95OuapEooAynmwqK+B1+eCji9+67odqr6jOJVDvff/89b775Jp9++ilmxpo1azAzhgwZQuPGjVm6dOk662+22WZsv/32zJs3j59++qnUVsUFF1zAmDFj1nm+d+/eDBo0aK3nmjRpwvz58/94nJ+fT5MmTdZ57dZbbw1AvXr1OO200yq147o41eLUU9pmzgznp7p1KxyqnZenJCFSTY0YMYKTTz6ZuXPn8tVXXzF//nxatmzJ22+/TatWrVi4cCEzZswAYO7cuUyZMoV27dqx/vrr07dvXwYOHMhvv/0GwOLFi3nmmWfW2ccdd9zB5MmT17kVTRIARx11FE8++SQrV65kzpw5fPHFF3To0GGd9b7++msgjLQeOXIkbdq0yeRhKbOakShWr4bBg0MRv08/hX//G/bfP9tRiUjChg8fzjHHHLPWcz179mT48OHUq1eP//znP5x22mm0a9eOXr168cADD9CoUSMAbrzxRjbffHNat25NmzZt6N69e1p9FqnssssuHHvssbRu3ZquXbsydOjQP6546tatGwsXLgTgxBNPZNddd2XXXXflu+++48orrwRg0aJFNG3alNtvv50bb7yRpk2b8tNPP1UopnSYuye+k0zKy8vzCRMmrPVcly7hZ4mnng47DF57Df785zAmYqutkgxRRCIzZsxg5513znYYNU5xx93MJrp7Xnm2V2VbFKVe6bRiBaxZE+737x+mJn32WSUJEZEyqnKJ4vPPQ3IYMKCwm2GdK53efTc8OXRoeNyzZ7iJiEiZVbmrnn79NfwsthrssmVw+eXwr39B8+agJq9I1rm7CgNWoiS6E6pcomjQoIS+iLfeCkX85s2Dc86Bm26CDTes7PBEJKZ+/fosWbKExo0bK1lUgoL5KOrXr5/R7Va5RJHS+uuHqq/77JPtSESEMPI4Pz8/4/MjSMkKZrjLpCp31VPDhnn+88/RVU///S989lk43QSh81pjIkRE1pGzVz2ZWVcz+9zMZpnZOqNPzKyemT0VLf/QzFqkteFFi8Iscz17wnPPQTQgRklCRCTzEksUZlYbGAocDrQGjjez1kVW6wssdfftgTuAwaVtt9GqJaGT+sUXw2RC772nIn4iIglKskXRAZjl7rPd/TfgSaBHkXV6AI9E90cAB1kpPV5brpwLbdrAlCkwaFCYXEhERBKTZGd2E2B+7HE+0LGkddx9tZn9CDQGvouvZGb9gYILYVfaO+9MVaVXADajyLGqwXQsCulYFNKxKLRjeV9YJa56cvdhwDAAM5tQ3g6Z6kbHopCORSEdi0I6FoXMbELpaxUvyVNPC4BmscdNo+eKXcfM1gMaAUsSjElERMooyUQxHmhlZi3NrC7QGxhVZJ1RwKnR/V7Am17VrtcVEanmEjv1FPU5nAOMBmoDD7n7NDO7Hpjg7qOAB4HHzGwW8D0hmZRmWFIxV0E6FoV0LArpWBTSsShU7mNR5QbciYhI5apy1WNFRKRyKVGIiEhKOZsoEiv/UQWlcSwuNLPpZvaJmb1hZttmI87KUNqxiK3X08zczKrtpZHpHAszOzb625hmZk9UdoyVJY3/keZmNsbMJkX/J92yEWfSzOwhM/vWzKaWsNzM7K7oOH1iZu3T2rC759yN0Pn9JfAnoC4wBWhdZJ2zgXuj+72Bp7IddxaPxQHA+tH9s2rysYjWawiMAz4A8rIddxb/LloBk4BNosdbZDvuLB6LYcBZ0f3WwFfZjjuhY7E/0B6YWsLybsArgAF7AR+ms91cbVEkUv6jiir1WLj7GHdfHj38gDBmpTpK5+8C4AZC3bAVlRlcJUvnWJwBDHX3pQDu/m0lx1hZ0jkWDmwU3W8ELKzE+CqNu48jXEFakh7Aox58AGxsZluXtt1cTRTFlf9oUtI67r4aKCj/Ud2kcyzi+hK+MVRHpR6LqCndzN1fqszAsiCdv4sdgB3M7F0z+8DMulZadJUrnWNxLXCSmeUDLwPnVk5oOaesnydAFSnhIekxs5OAPKBztmPJBjOrBdwO9MlyKLliPcLppy6EVuY4M9vV3X/IalTZcTzwsLvfZmadCOO32rj779kOrCrI1RaFyn8USudYYGYHA1cAR7n7ykqKrbKVdiwaAm2AsWb2FeEc7Khq2qGdzt9FPjDK3Ve5+xxgJiFxVDfpHIu+wNMA7v4+UJ9QMLCmSevzpKhcTRQq/1Go1GNhZrsD9xGSRHU9Dw2lHAt3/9HdN3P3Fu7egtBfc5S7l7sYWg5L539kJKE1gZltRjgVNbsyg6wk6RyLecBBAGa2MyFR1MT5WUcBp0RXP+0F/OjuX5f2opw89eTJlf+octI8FkOADYFnov78ee5+VNaCTkiax6JGSPNYjAYONbPpwBrgEnevdq3uNI/FRcD9ZnYBoWO7T3X8YmlmwwlfDjaL+mOuAeoAuPu9hP6ZbsAsYDlwWlrbrYbHSkREMihXTz2JiEiOUKIQEZGUlChERCQlJQoREUlJiUJERFJSopCcZGZrzGxy7NYixbrLMrC/h81sTrSvj6PRu2XdxgNm1jq6f3mRZe9VNMZoOwXHZaqZvWBmG5eyfrvqWilVKo8uj5WcZGbL3H3DTK+bYhsPAy+6+wgzOxS41d3bVmB7FY6ptO2a2SPATHf/e4r1+xAq6J6T6Vik5lCLQqoEM9swmmvjYzP71MzWqRprZlub2bjYN+79oucPNbP3o9c+Y2alfYCPA7aPXnthtK2pZnZ+9NwGZvaSmU2Jnj8uen6smeWZ2S1AgyiOx6Nly6KfT5rZEbGYHzazXmZW28yGmNn4aJ6AAWkclveJCrqZWYfoPU4ys/fMbMdolPL1wHFRLMdFsT9kZh9F6xZXfVdkbdmun66bbsXdCCOJJ0e35whVBDaKlm1GGFla0CJeFv28CLgiul+bUPtpM8IH/wbR85cCVxezv4eBXtH9vwAfAnsAnwIbEEa+TwN2B3oC98de2yj6OZZo/ouCmGLrFMR4DPBIdL8uoZJnA6A/cGX0fD1gAtCymDiXxd7fM0DX6PFGwHrR/YOBZ6P7fYB/xV5/E3BSdH9jQv2nDbL9+9Ytt285WcJDBPjV3dsVPDCzOsBNZrY/8Dvhm/SWwKLYa8YDD0XrjnT3yWbWmTBRzbtReZO6hG/ixRliZlcSagD1JdQGes7df4li+C+wH/AqcJuZDSacrnq7DO/rFeBOM6sHdAXGufuv0emutmbWK1qvEaGA35wir29gZpOj9z8D+F9s/UfMrBWhREWdEvZ/KHCUmV0cPa4PNI+2JVIsJQqpKk4ENgf2cPdVFqrD1o+v4O7jokRyBPCwmd0OLAX+5+7Hp7GPS9x9RMEDMzuouJXcfaaFeS+6ATea2Rvufn06b8LdV5jZWOAw4DjCJDsQZhw7191Hl7KJX929nZmtT6ht9FfgLsJkTWPc/Zio439sCa83oKe7f55OvCKgPgqpOhoB30ZJ4gBgnXnBLcwV/o273w88QJgS8gNgHzMr6HPYwMx2SHOfbwNHm9n6ZrYB4bTR22a2DbDc3f9DKMhY3LzDq6KWTXGeIhRjK2idQPjQP6vgNWa2Q7TPYnmY0fA84CIrLLNfUC66T2zVnwmn4AqMBs61qHllofKwSEpKFFJVPA7kmdmnwCnAZ8Ws0wWYYmaTCN/W73T3xYQPzuFm9gnhtNNO6ezQ3T8m9F18ROizeMDdJwG7Ah9Fp4CuAW4s5uXDgE8KOrOLeI0wudTrHqbuhJDYpgMfm9lUQtn4lC3+KJZPCJPy/B9wc/Te468bA7Qu6MwmtDzqRLFNix6LpKTLY0VEJCW1KEREJCUlChERSUmJQkREUlKiEBGRlJQoREQkJSUKERFJSYlCRERS+n8UvuD+REI9qgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {
            "needs_background": "light",
            "tags": []
          },
          "output_type": "display_data"
        }
      ],
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, val_dataloader)\n",
        "\n",
        "# Evaluate the Bert classifier\n",
        "evaluate_roc(probs, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 26,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxHGJj7BfhWv",
        "outputId": "64a14bae-a574-4301-f1bf-261080522245"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  83\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.25      0.87      0.39       116\n",
            "           1       0.82      0.18      0.30       369\n",
            "\n",
            "    accuracy                           0.35       485\n",
            "   macro avg       0.54      0.53      0.35       485\n",
            "weighted avg       0.68      0.35      0.32       485\n",
            "\n",
            "Accuracy Score:\t 0.34845360824742266\n",
            "Precision Score:\t 0.8192771084337349\n",
            "Recall Score:\t\t0.1842818428184282\n",
            "F1 Score:\t 0.30088495575221236\n"
          ]
        }
      ],
      "source": [
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord5.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_val, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 28,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yu1vQFK_fhWv",
        "outputId": "ceec76e9-c0e1-459e-d744-b2399d65cc0b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Tokenizing data...\n",
            "Number of tweets predicted as Rumor:  572\n",
            "\n",
            "               precision    recall  f1-score   support\n",
            "\n",
            "           0       0.09      0.90      0.17       475\n",
            "           1       0.92      0.11      0.20      4752\n",
            "\n",
            "    accuracy                           0.18      5227\n",
            "   macro avg       0.51      0.51      0.18      5227\n",
            "weighted avg       0.84      0.18      0.19      5227\n",
            "\n",
            "Accuracy Score:\t 0.1827051846183279\n",
            "Precision Score:\t 0.9195804195804196\n",
            "Recall Score:\t\t0.11069023569023569\n",
            "F1 Score:\t 0.19759579263711494\n"
          ]
        }
      ],
      "source": [
        "rhi_data = pd.read_csv('data/_RHI_text.csv')\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv')\n",
        "X_test = rhi_data.text.values\n",
        "y_test = rhi_y.isRumor.values\n",
        "\n",
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord5.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH)) \n",
        "testing_process(bert_classifier, X_test, y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F08yfWtRAzty"
      },
      "source": [
        "# BERTweet - Embedding Only"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {},
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
          ]
        }
      ],
      "source": [
        "import torch\n",
        "from transformers import AutoModel, AutoTokenizer \n",
        "import emoji\n",
        "bertweet = AutoModel.from_pretrained(\"vinai/bertweet-base\")\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False, normalization=False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 36,
      "metadata": {},
      "outputs": [],
      "source": [
        "raw_text = pd.read_csv('./data/_PHEME_text.csv')\n",
        "y = pd.read_csv('./data/_PHEME_target.csv')\n",
        "data = pd.concat([raw_text.text, y], axis=1).reset_index(drop=True)\n",
        "val = pd.read_csv('data/_PHEMEext_text.csv')\n",
        "\n",
        "X_train = data.text.values\n",
        "y_train = data.target.values\n",
        "\n",
        "X_val = val.drop(['Event'],axis=1).text.values\n",
        "y_val = val.target.values\n",
        "\n",
        "rhi_data = pd.read_csv('data/_RHI_text.csv')\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv')\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 32,
      "metadata": {},
      "outputs": [],
      "source": [
        "def text_preprocessing(text): # Create a function to tokenize a set of texts\n",
        "\n",
        "    \"\"\"\n",
        "    - Remove entity mentions (eg. '@united')\n",
        "    - Correct errors (eg. '&amp;' to '&')\n",
        "    @param    text (str): a string to be processed.\n",
        "    @return   text (Str): the processed string.\n",
        "    \"\"\"\n",
        "    # Remove '@name'\n",
        "    # text = text.lower()\n",
        "    # text = re.sub(r'(@.*?)[\\s]', ' ', text)\n",
        "    # text = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', text) # 해쉬태그 띄우기\n",
        "\n",
        "    # text = re.sub(r\"http\\S+\", \"*\", text)  # http link -> '*'\n",
        "\n",
        "    # text = re.sub(r\"@\\S+\", \"@\", text)   # mention -> '@'\n",
        "    # text = re.sub(r\"@[^\\s]+\", \"@\", text)   # mention -> '@'\n",
        "\n",
        "    # sent = re.sub(r\"(#)(\\S+)\", r'\\1 \\2', sent)\n",
        "    # sent = re.sub(r'([^\\s\\w@#\\*]|_)+', '', sent) # Erasing Special Characters\n",
        "\n",
        "    # Replace '&amp;' with '&'\n",
        "    text = emoji.demojize(text)\n",
        "    text = re.sub(r\"http\\S+\", \"HTTPURL\", text)  # http link -> '*'\n",
        "    text = re.sub(r\"@\\S+\", \"@USER\", text)   # mention -> '@'\n",
        "    text = re.sub(r':[^:]*:', r' \\g<0>', text)  # http link -> '*'\n",
        "\n",
        "    # Remove trailing whitespace\n",
        "    text = re.sub(r'\\s+', ' ', text).strip()\n",
        "    text = re.sub(r'&amp;', '&', text)\n",
        "\n",
        "    # input_ids = torch.tensor([tokenizer.encode(text)])\n",
        "    text = text.split()\n",
        "    return text"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 33,
      "metadata": {},
      "outputs": [
        {
          "ename": "NameError",
          "evalue": "name 'pheme_text' is not defined",
          "output_type": "error",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-33-7d21959e8c67>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mpheme_text\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m: name 'pheme_text' is not defined"
          ]
        }
      ],
      "source": [
        "pheme_text"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "## Class"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "5P3_E6mOoRqF"
      },
      "outputs": [],
      "source": [
        "class BertTweetClassifier(nn.Module): # Create the BertClassfier class\n",
        "    \"\"\"Bert Model for Classification Tasks.\n",
        "    \"\"\"\n",
        "\n",
        "    def __init__(self, freeze_bert=False):\n",
        "        \"\"\"\n",
        "        @param    bert: a BertModel object\n",
        "        @param    classifier: a torch.nn.Module classifier\n",
        "        @param    freeze_bert (bool): Set `False` to fine-tune the BERT model\n",
        "        \"\"\"\n",
        "        super(BertTweetClassifier, self).__init__()\n",
        "        # Specify hidden size of BERT, hidden size of our classifier, and number of labels\n",
        "        D_in, H, D_out = 768, 50, 2\n",
        "\n",
        "        # Instantiate BERT model\n",
        "        self.bert = AutoModel.from_pretrained(\"vinai/bertweet-base\")\n",
        "\n",
        "\n",
        "        # Instantiate an one-layer feed-forward classifier\n",
        "        self.classifier = nn.Sequential(\n",
        "            nn.Linear(D_in, H),\n",
        "            nn.ReLU(),\n",
        "            # nn.Dropout(0.5),\n",
        "            nn.Linear(H, D_out)\n",
        "        )\n",
        "\n",
        "        # Freeze the BERT model\n",
        "        if freeze_bert:\n",
        "            for param in self.bert.parameters():\n",
        "                param.requires_grad = False\n",
        "\n",
        "    def forward(self, input_ids, attention_mask):\n",
        "        \"\"\"\n",
        "        Feed input to BERT and the classifier to compute logits.\n",
        "        @param    input_ids (torch.Tensor): an input tensor with shape (batch_size,\n",
        "                      max_length)\n",
        "        @param    attention_mask (torch.Tensor): a tensor that hold attention mask\n",
        "                      information with shape (batch_size, max_length)\n",
        "        @return   logits (torch.Tensor): an output tensor with shape (batch_size,\n",
        "                      num_labels)\n",
        "        \"\"\"\n",
        "        # Feed input to BERT\n",
        "        outputs = self.bert(input_ids=input_ids,\n",
        "                            attention_mask=attention_mask)\n",
        "\n",
        "        # Extract the last hidden state of the token `[CLS]` for classification task\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        # Feed input to classifier to compute logits\n",
        "        logits = self.classifier(last_hidden_state_cls)\n",
        "\n",
        "        return logits\n",
        "\n",
        "    def embedding(self, input_ids):\n",
        "        outputs = self.bert(input_ids=input_ids)\n",
        "        last_hidden_state_cls = outputs[0][:, 0, :]\n",
        "\n",
        "        return last_hidden_state_cls  # sequence_output, pooled_output, (hidden_states), (attentions)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "M181HsetpgSU"
      },
      "source": [
        "## Execution"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "v334DoYDfhWs"
      },
      "outputs": [],
      "source": [
        "raw_text = pd.read_csv('./data/_PHEME_text.csv')\n",
        "y = pd.read_csv('./data/_PHEME_target.csv')\n",
        "data = pd.concat([raw_text.text, y], axis=1).reset_index(drop=True)\n",
        "val = pd.read_csv('data/_PHEMEext_text.csv')\n",
        "\n",
        "X_train = data.text.values\n",
        "y_train = data.target.values\n",
        "\n",
        "X_val = val.drop(['Event'],axis=1).text.values\n",
        "y_val = val.target.values\n",
        "\n",
        "rhi_data = pd.read_csv('data/_RHI_text.csv').text.values\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv').isRumor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "OwRCb2rGsFQ8",
        "outputId": "581cc42b-2d21-4f0d-d0db-5736ad4fa1ad"
      },
      "outputs": [
        {
          "data": {
            "application/x.notebook.stream": "(5802,) (5802,)\n(485,) (485,)\n"
          },
          "output_type": "unknown"
        }
      ],
      "source": [
        "print(X_train.shape, y_train.shape)\n",
        "print(X_val.shape, y_val.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 258,
          "referenced_widgets": [
            "3a7e702d08a4446ea50b410f2e9843ab",
            "ce795ec9283d46b8927ddc7dc93a2d32",
            "3111d583f32540d3a18bb468741ff888",
            "d202a9f02ebf442e88967fd5d6ddd303",
            "7d0cbaf5fa69439e836c0a49b53b3b42",
            "feb22d4ea5124d13aa49a849eee755f1",
            "72f3f4bbc9d740d4b4c099f8890c84c8",
            "69415ec2eff448f09502f6bb35dd6345",
            "a0e9851bf5b64d13abb59af9b7918082",
            "0ab9fc781a51454ca816230c73d55538",
            "10b4a6adb70440168442d2748c249915",
            "e5a6031cb1d04c4abd7ec69e91eabbe8",
            "b96f5ef0297f4936bd403f02198c50b8",
            "07a9ca0cf13f462f830bba75a4910399",
            "2fa3a91087fb4213b65d8aea738b3c9a",
            "375cc6ce64ce4791835ce34cb3bc2b8c",
            "86ae2530281341bcb7038f6fc347f8b8",
            "772efbe2ccc143f4b182348720311d9e",
            "cf606c73bf674f11b1a6d6db97fc5fff",
            "f1759ca550bf44d0a33c3d86a4347154",
            "af99588ee18e413ba551a3ffd68861de",
            "9c9dce7a03fb4e9bb773a8e49b399fcc",
            "c6ae66940c8d4d6cb1f1a9d1e66c8a2b",
            "2cb51beb13da4c3688008a052ab82ee3"
          ]
        },
        "id": "4ZUhevayfhWu",
        "outputId": "e9b5d796-079f-4433-ac42-31205d4e8de6"
      },
      "outputs": [
        {
          "data": {
            "application/x.notebook.stream": "No GPU available, using the CPU instead.\n"
          },
          "output_type": "unknown"
        },
        {
          "data": {
            "application/x.notebook.stream": "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
          },
          "output_type": "unknown"
        },
        {
          "data": {
            "application/x.notebook.stream": "Max length:  58\n\nTokenizing data...\n"
          },
          "output_type": "unknown"
        }
      ],
      "source": [
        "device = getDevice()\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
        "train_dataloader, val_dataloader = data_process(X_train, y_train, X_val, y_val, batch_size=8)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "x-HVi7I2uVey",
        "outputId": "b0f83d49-358d-420f-d664-d4b40a576f6d"
      },
      "outputs": [
        {
          "data": {
            "application/x.notebook.stream": "There are 1 GPU(s) available.\nDevice name: Tesla T4\n"
          },
          "output_type": "unknown"
        }
      ],
      "source": [
        "if torch.cuda.is_available():       \n",
        "      device = torch.device(\"cuda\")\n",
        "      print(f'There are {torch.cuda.device_count()} GPU(s) available.')\n",
        "      print('Device name:', torch.cuda.get_device_name(0))\n",
        "\n",
        "else:\n",
        "    print('No GPU available, using the CPU instead.')\n",
        "    device = torch.device(\"cpu\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Vtfc0vdYfhWu",
        "outputId": "e3c09e56-86a9-4cc1-808e-b1fd61430657"
      },
      "outputs": [
        {
          "data": {
            "application/x.notebook.stream": "Start training...\n\n Epoch  |  Batch  |  Train Loss  |  Val Loss  |  Val Acc  |  Elapsed \n----------------------------------------------------------------------\n   1    |   20    |   0.657699   |     -      |     -     |   4.73   \n   1    |   40    |   0.572159   |     -      |     -     |   4.54   \n   1    |   60    |   0.599299   |     -      |     -     |   4.65   \n   1    |   80    |   0.502215   |     -      |     -     |   4.70   \n   1    |   100   |   0.424222   |     -      |     -     |   4.74   \n   1    |   120   |   0.471874   |     -      |     -     |   4.66   \n   1    |   140   |   0.555176   |     -      |     -     |   4.64   \n   1    |   160   |   0.485429   |     -      |     -     |   4.55   \n   1    |   180   |   0.442740   |     -      |     -     |   4.51   \n   1    |   200   |   0.574039   |     -      |     -     |   4.47   \n   1    |   220   |   0.487704   |     -      |     -     |   4.47   \n   1    |   240   |   0.527812   |     -      |     -     |   4.45   \n   1    |   260   |   0.427125   |     -      |     -     |   4.44   \n   1    |   280   |   0.511334   |     -      |     -     |   4.43   \n   1    |   300   |   0.385671   |     -      |     -     |   4.40   \n   1    |   320   |   0.463129   |     -      |     -     |   4.42   \n   1    |   340   |   0.411448   |     -      |     -     |   4.40   \n   1    |   360   |   0.424775   |     -      |     -     |   4.43   \n   1    |   380   |   0.455251   |     -      |     -     |   4.43   \n   1    |   400   |   0.397658   |     -      |     -     |   4.45   \n   1    |   420   |   0.486652   |     -      |     -     |   4.47   \n   1    |   440   |   0.442327   |     -      |     -     |   4.49   \n   1    |   460   |   0.430557   |     -      |     -     |   4.50   \n   1    |   480   |   0.434291   |     -      |     -     |   4.51   \n   1    |   500   |   0.459816   |     -      |     -     |   4.51   \n   1    |   520   |   0.394905   |     -      |     -     |   4.52   \n   1    |   540   |   0.542678   |     -      |     -     |   4.60   \n   1    |   560   |   0.493274   |     -      |     -     |   4.54   \n   1    |   580   |   0.518410   |     -      |     -     |   4.49   \n   1    |   600   |   0.466646   |     -      |     -     |   4.48   \n   1    |   620   |   0.421744   |     -      |     -     |   4.46   \n   1    |   640   |   0.436818   |     -      |     -     |   4.47   \n   1    |   660   |   0.485056   |     -      |     -     |   4.46   \n   1    |   680   |   0.715749   |     -      |     -     |   4.45   \n   1    |   700   |   0.601952   |     -      |     -     |   4.43   \n   1    |   720   |   0.416572   |     -      |     -     |   4.43   \n   1    |   725   |   0.777217   |     -      |     -     |   1.00   \n----------------------------------------------------------------------\n   1    |    -    |   0.489019   |  0.858612  |   62.62   |  167.24  \n----------------------------------------------------------------------\n\n\nTraining complete!\n"
          },
          "output_type": "unknown"
        }
      ],
      "source": [
        "set_seed(42)    # Set seed for reproducibility\n",
        "bert_classifier, optimizer, scheduler, loss_fn = initialize_model(epochs=4)\n",
        "train(bert_classifier, train_dataloader, val_dataloader, epochs=1, evaluation=True)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "xqI9LRVbfhWu"
      },
      "outputs": [],
      "source": [
        "torch.save(bert_classifier.state_dict(), './Model/BERTweet_raw_to_fine_tune_ord4.pt')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 330
        },
        "id": "1E0J9rlrfhWu",
        "outputId": "94c6ed92-673a-473e-a3c8-1cd4973459bf"
      },
      "outputs": [
        {
          "data": {
            "application/x.notebook.stream": "AUC: 0.4855\nAccuracy: 62.89%\n"
          },
          "output_type": "unknown"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYoAAAEWCAYAAAB42tAoAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd5gUVdbH8e8hIyIquAYQZQUUUIKMYCSIARFBF0RkRVEEs5jYNQdeVtfVNa2uAuJiBBUTRtSVYBYQJIoiLjAgiogKIkg47x+3hmnGmZ5mmO6e7vl9nqef6Qpddbpmpk/fulXnmrsjIiJSlArpDkBERMo2JQoREYlLiUJEROJSohARkbiUKEREJC4lChERiUuJQraJmc0xsw7pjqOsMLNrzezhNO17lJkNTce+S5uZ/dnM3izha/U3mWRKFBnMzP5nZr+a2RozWx59cOyYzH26ezN3n5jMfeQxs6pmdpuZLY7e55dmNtjMLBX7LySeDmaWGzvP3W9193OTtD8zs0vNbLaZ/WJmuWb2rJkdlIz9lZSZ3WxmT2zPNtz9SXc/LoF9/S45pvJvsrxSosh8J7n7jkBLoBVwTZrj2WZmVqmIRc8CnYAuQE2gLzAQuDcJMZiZlbX/h3uBQcClwK5AY+BF4MTS3lGc30HSpXPfkiB31yNDH8D/gGNipv8BvBozfSjwAfAj8BnQIWbZrsB/gGXAKuDFmGVdgRnR6z4AmhfcJ7AX8Cuwa8yyVsD3QOVo+hxgXrT98cA+Mes6cBHwJfB1Ie+tE7AO2LvA/LbAJqBhND0RuA34BPgZeKlATPGOwUTgb8D70XtpCJwdxbwaWAicF61bI1pnM7AmeuwF3Aw8Ea2zb/S+zgIWR8fiupj9VQcejY7HPOAvQG4Rv9tG0ftsE+f3Pwp4AHg1ivdjYL+Y5fcCS6LjMg04KmbZzcBY4Ilo+blAG+DD6Fh9A9wPVIl5TTPgLeAH4FvgWqAz8BuwITomn0Xr1gJGRttZCgwFKkbL+kXH/G5gZbSsH/BetNyiZd9Fsc0CDiR8SdgQ7W8N8HLB/wOgYhTXV9ExmUaBvyE9SvBZk+4A9NiOX97W/yD1on+oe6PputE/YRdCy/HYaHq3aPmrwNPALkBloH00v1X0D9o2+qc7K9pP1UL2+Q4wICaeO4CHoufdgQVAE6AScD3wQcy6Hn3o7ApUL+S9/R2YVMT7XkT+B/jE6IPoQMKH+XPkf3AXdwwmEj7Qm0UxViZ8W98v+rBqD6wFDo7W70CBD3YKTxQjCEmhBbAeaBL7nqJjXg+YWXB7Mds9H1hUzO9/VPR+2kTxPwmMiVl+BlA7WnYlsByoFhP3BuDk6NhUB1oTEmul6L3MAy6L1q9J+NC/EqgWTbcteAxi9v0CMCz6nfyBkMjzfmf9gI3AJdG+qrN1ojie8AG/c/R7aALsGfOeh8b5PxhM+D/YP3ptC6B2uv9XM/2R9gD02I5fXvgHWUP45uTAf4Gdo2V/BR4vsP54wgf/noRvxrsUss0Hgf8rMG8++Ykk9p/yXOCd6LkRvr22i6ZfB/rHbKMC4UN3n2jagaPjvLeHYz/0Ciz7iOibOuHD/u8xy5oSvnFWjHcMYl47pJhj/CIwKHregcQSRb2Y5Z8AvaPnC4HjY5adW3B7McuuAz4qJrZRwMMx012Az+OsvwpoERP35GK2fxnwQvT8dGB6EettOQbR9O6EBFk9Zt7pwIToeT9gcYFt9CM/URwNfEFIWhUKec/xEsV8oHsy/t/K86OsnZOVbXeyu9ckfIgdANSJ5u8DnGpmP+Y9gCMJSWJv4Ad3X1XI9vYBrizwur0Jp1kKeg44zMz2BNoRks+7Mdu5N2YbPxCSSd2Y1y+J876+j2ItzJ7R8sK2s4jQMqhD/GNQaAxmdoKZfWRmP0TrdyH/mCZqeczztUDeBQZ7FdhfvPe/kqLffyL7wsyuMrN5ZvZT9F5qsfV7KfjeG5vZK9GFET8Dt8asvzfhdE4i9iH8Dr6JOe7DCC2LQvcdy93fIZz2egD4zsyGm9lOCe57W+KUBClRZAl3n0T4tnVnNGsJ4dv0zjGPGu7+92jZrma2cyGbWgL8rcDrdnD30YXscxXwJnAa0IfQAvCY7ZxXYDvV3f2D2E3EeUtvA23NbO/YmWbWlvBh8E7M7Nh16hNOqXxfzDH4XQxmVpWQ/O4Ednf3nYHXCAmuuHgT8Q3hlFNhcRf0X6CemeWUZEdmdhShD6QXoeW4M/AT+e8Ffv9+HgQ+Bxq5+06Ec/156y8B/ljE7gpuZwmhRVEn5rjv5O7N4rxm6w263+furQktxMaEU0rFvi7a937FrCPbSIkiu9wDHGtmLQidlCeZ2fFmVtHMqkWXd9Zz928Ip4b+bWa7mFllM2sXbWMEcL6ZtY2uBKphZieaWc0i9vkUcCbQM3qe5yHgGjNrBmBmtczs1ETfiLu/TfiwfM7MmkXv4dDofT3o7l/GrH6GmTU1sx2AIcBYd98U7xgUsdsqQFVgBbDRzE4AYi/Z/BaobWa1En0fBTxDOCa7mFld4OKiVoze37+B0VHMVaL4e5vZ1QnsqyahH2AFUMnMbgSK+1Zek9B5vMbMDgAuiFn2CrCnmV0WXbZcM0raEI7LvnlXjUV/X28C/zSzncysgpntZ2btE4gbMzsk+vurDPxCuKhhc8y+ikpYEE5Z/p+ZNYr+fpubWe1E9itFU6LIIu6+AngMuNHdlxA6lK8lfFgsIXwry/ud9yV88/6c0Hl9WbSNqcAAQtN/FaFDul+c3Y4jXKGz3N0/i4nlBeB2YEx0GmM2cMI2vqUewATgDUJfzBOEK2kuKbDe44TW1HJCR+ulUQzFHYOtuPvq6LXPEN57n+j95S3/HBgNLIxOqRR2Oi6eIUAu8DWhxTSW8M27KJeSfwrmR8IplVOAlxPY13jCcfuCcDpuHfFPdQFcRXjPqwlfGJ7OWxAdm2OBkwjH+UugY7T42ejnSjP7NHp+JiHxziUcy7EkdioNQkIbEb1uEeE03B3RspFA0+j4v1jIa+8i/P7eJCS9kYTOctkOln+mQCTzmNlEQkdqWu6O3h5mdgGhozuhb9oi6aIWhUiKmNmeZnZEdCpmf8Klpi+kOy6R4iQtUZjZI2b2nZnNLmK5mdl9ZrbAzGaa2cHJikWkjKhCuPpnNaEz/iVCP4RImZa0U09R5+ga4DF3P7CQ5V0I55q7EG7uutfd2xZcT0RE0itpLQp3n0y4dr4o3QlJxN39I2Dn6Hp8EREpQ9JZjKsuW1+FkRvN+6bgimY2kFDnhRo1arQ+4IADUhKgiEjGW7SIjSt/5DPf+L2771aSTWRE1UZ3Hw4MB8jJyfGpU6emOSIRkbJh+HB46qkCM/O6FMzotsuDrF/9HZ+tv3lRSfeRzkSxlK3vTK0XzRMRkThik8OkSeFn++gi6zrrl3L5lxcwYbfTeHv3PzNurwtC8ZhJN5d4f+lMFOOAi81sDKEz+6fojk4REYkU1mKITQ7t20OfPjBwgMPDD8NVV8GGDRxx2Ylcf17+a7ZnuK+kJQozG00oVFfHwqhgNxEKheHuDxFq6HQh3Pm7ljAOgIhIuRevxZD3vE8fGDgwmvHVV9BpAEyYAB07wogRsF/plbxKWqJw99OLWe6EgWtERCTGU0/BjBnQsmUhSaEws2bBtGkhw5x77vY1HwqREZ3ZIiLZLrYVkZckJk6M84LZs+HTT+HMM+Hkk2HhQqidnPqHKuEhIlIG5LUiICSJPn2KWPG33+Dmm+Hgg+G662DdujA/SUkC1KIQEUmpQi9nJcFWxMcfQ//+MGcOnHEG3H03VKuWrFC3UKIQEUmy4jqnoZhWBMDSpXDUUbD77vDKK3DiiUmJtTBKFCIiSVBUckioczrWF19A48ZQty48/TR06gQ7JToybOlQohARKSWllhwAfvwR/vKXcG/ExInQrh2cckpph5wQJQoRkVKyzZe1FmXcOLjgAli+HAYPhkMOKfVYt4UShYhIKSq2Q7o4554LI0fCQQfBSy9BTk5phVZiShQiIukWU8SPnBzYZx/461+hSpX0xhVRohAR2Q6F3Si3TZYsgfPPh969oW/f8LyM0Q13IiLbIeEb5QravBkefBCaNQvnqtavT1aI200tChGR7bTN/RJffhn6IiZPhmOOCc2SBg2SFd52U6IQEUm1uXNh5kx45BHo16/Ui/iVNiUKEZFU+OyzcI7qrLOge/dQxG+XXdIdVULURyEikkzr18MNN4SrmW64Ib+IX4YkCVCiEBFJng8/hFatYOjQ0Ms9fXpKiviVNp16EhFJhqVLw+3Ze+wBr70GJ5yQ7ohKTC0KEZHSNG9e+Fm3LjzzTCgJnsFJApQoRERKx6pVcM450LQpvPtumHfyyVCzZnrjKgVKFCIiJTB8OHToEC5kOvL7F0KCeOwxuOaatBfxK23qoxARSVBhZcRf3eMcusz5T7jr7tVXwxClWUaJQkQkQVvKiLdw2reDPn82unAorGwEV10FlSunO8SkUKIQEYmjYNG/4w9YxNPVzwuXu555JlCSAScyixKFiEgBhZ1i6tBuMzfv9iAXzrgaKjmcemr6AkwxdWaLiBQQWxG2fXsYc8t8Jmxuz2ULLqZK+8Nh9mzo3z+9QaaQWhQiIoXYqiLsuPlwzxwYNSqcbirjRfxKmxKFiJRbsaeYYs2YAT33mw7/mQFnnw3duoUifjvvnPogywCdehKRciv2FFOeKpvXcf9O1zJ8xiFw8835RfzKaZIAtShEpJzb6hTT+++Hvocl80NL4p//zMgifqVNiUJEBEIRv44dQ42m8ePhuOPSHVGZoUQhIuVKwfsiujeaCzQNCeK550Ky2HHHtMZY1qiPQkTKlbx+iZobfmBMtX48OrVZGLsa4KSTlCQKoRaFiJQ7l9Z9jiFfXQQrV8J110GbNukOqUxTohCRrBd7umngB/3os+HRULzvjTdCb7bEpVNPIpL1nnrSmTHdAVi27+F8fMrf4eOPlSQSlNQWhZl1Bu4FKgIPu/vfCyyvDzwK7Bytc7W7v5bMmESknPn6a+6cOZC39jyDayaeRXko4lfaktaiMLOKwAPACUBT4HQza1pgteuBZ9y9FdAb+Hey4hGRcmbTJrjvPjjwQJqu/gjD0x1Rxkrmqac2wAJ3X+juvwFjgO4F1nFgp+h5LWBZEuMRkfJi3jw46igYNAjat6dfzhze2KNfuqPKWMlMFHWBJTHTudG8WDcDZ5hZLvAacElhGzKzgWY21cymrlixIhmxikg2WbCAdTPn87cDHqfDL6/y5uf10x1RRkt3Z/bpwCh3rwd0AR43s9/F5O7D3T3H3XN22223lAcpIhlg2jR45JHw/KST+FPLr7njmzPAjJYtwzhDUjLJ7MxeCuwdM10vmherP9AZwN0/NLNqQB3guyTGJSLZ5Ndf4ZZb4M47Ye+9Q0aoVo21lXbauo6TlFgyWxRTgEZm1sDMqhA6q8cVWGcx0AnAzJoA1QCdWxKRxEyeDC1awO23Q79+MH26ivglQdJaFO6+0cwuBsYTLn19xN3nmNkQYKq7jwOuBEaY2eWEju1+7q5LE0SkeEuXQqdOoRXx9tvhuSRFUu+jiO6JeK3AvBtjns8FjkhmDCKSZWbNgoMOCkX8XnghFPGrUSPdUWW1dHdmi4gk5vvvoW9faN48v4hf165KEimgRCEiZZs7PPMMNG0KY8bATTdB27ZFrj58OHTo8PuR66TkVBRQRMq2s86Cxx+HnBz473/DaacCYov+TZoUfrZvr0tiS4sShYiUPXnXtJiFT/zmzeGyy6BS/kdWUckhL0EMVEmnUqNEISJly8KFMGAAnHEGnH02wzf156lXgFe2Xk3JIXWUKESkTBjx0CbW3/kv+n99HZutIvctPZPxj26dEGIpOaSOEoWIpN/cuRzxl3NouvpjPtz1RO5q/BArqtYDlBDKAiUKEUmbvH6GQ1d+zVVrvmJIk6e4cU5vnjVLd2gSQ4lCRNJjyhR++ucMZnw7AFqeSN/DF3LKmTVBOaLMUaIQkdRauxZuvBHuvps/V9mHtw7py5sTqwE10x2ZFEE33IlISgwfDpe1nMjS2s3hn/9k3O4DOLTKdH6roCJ+ZZ1aFCKSEm/9J5enPjuW76rtw2XN32HGLh35I7opLhMoUYhIcn32GbRowYqq9bj+wJe4/eMO3LPDDumOSraBTj2JSHKsWBGaCy1bbrkZ4uPaXUBJIuOoRSEipcs9FO+79FL46acw+txhh6U7KtkOShQiUrr69oUnnwwVXkeOhGbN0h2RbKeEE4WZ7eDua5MZjIhkqM2bQwE/szCQUOvWoUVRsWK6I5NSUGwfhZkdbmZzgc+j6RZm9u+kRyYimWHBgjAM6X/+E6b794fLL4eKFbeMDaHxITJbIp3ZdwPHAysB3P0zoF0ygxKRDLBxI9x5ZxgfYvp0qFIFYKvkcN55+UX9WrbUpbCZKqFTT+6+xLauvbIpOeGISEaYPRvOPhumToXu3eHf/4a99gJC7aYZM0JiUEG/7JBIolhiZocDbmaVgUHAvOSGJSJl2uLFsGhRuLqpV6/QNxGjZUuYODE9oUnpSyRRnA/cC9QFlgJvAhcmMygRKYM+/jjcPDdwIHTpEgYY2nHHdEclKZBIH8X+7v5nd9/d3f/g7mcATZIdmIiUEb/8AldcEe6F+Mc/YP36MD8mSajTOrslkij+leA8Eck277wTxqu++244/3z49FOoWvV3q+X1S4A6rbNRkaeezOww4HBgNzO7ImbRToAujhbJdrm5cPzx0KBBuHSp3dYXO+YNOgT5ndfql8hO8fooqgA7RuvEFor/GeiZzKBEJI2mT4dWraBePXj55XDpUvXqwNbJIXYsa7Uispu5e/wVzPZx90UpiqdYOTk5PnXq1HSHIZJ9vv023E39zDOhadC+PVB0cgBd+ppJzGyau+eU5LWJXPW01szuAJoBW0YYcfejS7JDESlj3ENtpkGDYM0aGDoUDj98y2LdFyGJJIongaeBroRLZc8CViQzKBFJnQVt+9Bwyhhm73QY/2g+ksVvNYG38per/0ESSRS13X2kmQ1y90nAJDObkuzARCSJYor4jf35OFZVO4wpLS9is/3+OhX1P0giiWJD9PMbMzsRWAbsmryQRCSpvvgCBgyAM8+E/v15Y4+zYQ+1GKRoiSSKoWZWC7iScP/ETsBlSY1KRErfxo1w111w001QrdqWK5lEilNsonD3V6KnPwEdAczsiGQGJSKlbOZMOOccmDYNTjkFHngA9twz3VFJhoh3w11FoBehxtMb7j7bzLoC1wLVgVapCVFEtltuLixZAs8+Cz16/K6In0g88Up4jATOBWoD95nZE8CdwD/cPaEkYWadzWy+mS0ws6uLWKeXmc01szlm9tS2vgERKcIHH8BDD4XneUX8evZUkpBtFu/UUw7Q3N03m1k1YDmwn7uvTGTDUYvkAeBYIBeYYmbj3H1uzDqNgGuAI9x9lZn9oaRvREQia9bAddfBv/4F++0Xxo2oWhVq1NiySmHlN0SKEq9F8Zu7bwZw93XAwkSTRKQNsMDdF7r7b8AYoHuBdQYAD7j7qmg/323D9kWkoDffhAMPDEniootUxE9KRbwWxQFmNjN6bsB+0bQB7u7Ni9l2XWBJzHQu0LbAOo0BzOx9QqHBm939jYIbMrOBwECA+vXrF7NbkXJqyRI48cTQipg8GY48Mu7quolOEhUvUaRizIlKQCOgA1APmGxmB7n7j7EruftwYDiEWk8piEskc0ybBq1bw957w2uvwVFHhctfRUpJkaee3H1RvEcC214K7B0zXS+aFysXGOfuG9z9a+ALQuIQkeIsXw6nngo5OfnV+o49tsgkocGFpKQSueGupKYAjcysASFB9AYKngl9ETgd+I+Z1SGcilqYxJhEMp87PPYY6y68nAq/rmVUg1t5+obD2VTMMGQqCy4llbRE4e4bzexiYDyh/+ERd59jZkOAqe4+Llp2nJnNBTYBg7exw1ykXBk+HBpe35ujVzzDVI7gXB5mj/oHJPRaVX6Vkip2PAoAM6sO1Hf3+ckPKT6NRyHlUlTEr0NHo8knj3JA3dW8uNeFnP7nCvrgl4Rsz3gUxY6ZbWYnATOAN6LplmY2riQ7E5ES+PzzMAzpyJEAzGtzFoO+vJgJk5QkJDWKTRTAzYR7In4EcPcZQIMkxiQiABs2wK23QosWrJs+lyF37ahOaEmLhMqMu/tPtvVt/7pEVSSZZszg+25nU2fJDCbW6Unv7//Ft/P22NLPIJJKiSSKOWbWB6gYldy4FPgguWGJlHPLl2PfLqfvDs+xpNmfOAAYoo5oSZNEEsUlwHXAeuApwpVKQ5MZlEi59N57oRz4hRdC5870afMV6yvuoLunJe0S6aM4wN2vc/dDosf1Ue0nESkNq1fDxReHO6rvuQfWrwdgfcUd0hyYSJBIi+KfZrYHMBZ42t1nJzkmkfJj/PhwPmnJEhg0iEf+OJTHjg9F/FTVVcqKYlsU7t6RMLLdCmCYmc0ys+uTHplItluyBLp2hR12CKed7rmHx57fUVVdpcxJ6M5sd19OGLxoAvAX4EbUTyGy7dxhyhRo0yYU8Xv99VDlNaY+k6q6SlmTyA13TczsZjObBfyLcMVTvaRHJpJtvvkmDEPatm1+4aVjjlGlVynzEmlRPAI8DRzv7suSHI9I9nGHUaPgiitg3Tq4/XY44oh0RyWSsGIThbsflopARLJWr14wdmy4qunhh6Fx43RHJLJNikwUZvaMu/eKTjnF3omd6Ah3IuXXpk1gBhUqwEknwdFHw3nnhWmRDBOvRTEo+tk1FYGIZI1586B/fzj7bBgwAM48M+7qw4eHMaxBl8RK2RRvhLtvoqcXFjK63YWpCU8kg2zYAEOHhk/6+fOhVq2EXvbUU+iSWCnTEunMPhb4a4F5JxQyT6T8mj4d+vULJThOOw3uuw/+8IeEX65LYqUsi9dHcQGh5fBHM5sZs6gm8H6yAxPJKN9+C99/Dy++CN27pzsakVIVr0XxFPA6cBtwdcz81e7+Q1KjEskEkyfDrFlw0UXQuTMsWADVqyf0UvVLSCaJlyjc3f9nZhcVXGBmuypZSLn1889w9dXw4IPhUtdzz4WqVYtNErHJIe9+u/bt1S8hZV9xLYquwDTC5bGxIxc58MckxiVSNr32WrjMddmycAPdkCEhSSQgr9O6ZUu2DECk8SUkExSZKNy9a/RTw56KQCji17077L9/uIGubdtt3oQ6rSUTJVLr6QgzqxE9P8PM7jKz+skPTaQMcIePPgrP994b3nwTPv004SQxfDh06BAeGu9aMlUit4k+CKw1sxbAlcBXwONJjUqkLFi2DE4+GQ47LL9ToWNHqFIl4U3oHgnJBoncR7HR3d3MugP3u/tIM+uf7MBE0sYdRo6Eq64Ko83deed2FfHT6SbJdIkkitVmdg3QFzjKzCoAlZMblkga9ewJzz8fepwffhgaNkx3RCJplcipp9OA9cA50QBG9YA7khqVSKpt2gSbN4fnJ58MDz0E77yjJCFCYkOhLgeeBGqZWVdgnbs/lvTIRFJl9uxwamnkyDDdt68qvYrESOSqp17AJ8CpQC/gYzPrmezARJLut9/gllvg4IPhq69gl13SHZFImZRIH8V1wCHu/h2Ame0GvA2MTWZgIkk1bVoo4jd7drgU6Z57YLfdSmXTKs8h2SaRtnWFvCQRWZng60TKrpUr4ccf4eWX4cknSy1JgC6JleyTSIviDTMbD4yOpk8DXkteSCJJMmFCKOJ36aVw3HHw5ZdQrVpSdqVLYiWbJNKZPRgYBjSPHsPdXWNRSOb46afQOX300aGQ3/r1YX6SkoRItok3HkUj4E5gP2AWcJW7L01VYCKl4uWX4fzzYfnycAPdLbckXMRvW6hfQrJZvBbFI8ArQA9CBdl/pSQikdKyZAn06AG1a4d6TXfcATvsUKq7yKvldN55+VU+1C8h2SZeH0VNdx8RPZ9vZp+mIiCR7eIOH34Ihx+eX8Tv8MO3qT5TcYoaV0JlwyVbxWtRVDOzVmZ2sJkdDFQvMF0sM+tsZvPNbIGZXR1nvR5m5maWs61vQGSL3Fzo1i3cPJf3Cd6hQ6kkidgqsLGth/btYdiw0HGtJCHZKl6L4hvgrpjp5THTDhwdb8NmVhF4ADgWyAWmmNk4d59bYL2awCDg420LXSSyeTOMGAGDB8PGjXDXXXDkkaW6Cw06JOVZvIGLOm7nttsAC9x9IYCZjQG6A3MLrPd/wO3A4O3cn5RXPXrAiy+Gq5pGjIA/JmfwRV3yKuVVMm+cqwssiZnOjeZtEZ3C2tvdX423ITMbaGZTzWzqihUrSj9SyTwbN+YX8evRIySIt99OWpIQKc/Sdod1VK78LsJgSHG5+3B3z3H3nN1K8Q5ayVAzZ4bBhEZE11qccQacey6YxX+diJRIMhPFUmDvmOl60bw8NYEDgYlm9j/gUGCcOrSlSOvXw003QevWsGhRqZbdEJGiJVI91qKxsm+MpuubWZsEtj0FaGRmDcysCtAbGJe30N1/cvc67r6vu+8LfAR0c/epJXonkt2mTAlVXocMgdNPh3nz4E9/SndUIuVCIi2KfwOHAadH06sJVzPF5e4bgYuB8cA84Bl3n2NmQ8ysWwnjlfJq1SpYswZeew0eeyzcRCciKZFIUcC27n6wmU0HcPdVUQuhWO7+GgUKCLr7jUWs2yGRbUo58s47oYjfoEGhiN8XXySl/IaIxJdIi2JDdE+Ew5bxKDYnNSop3378EQYMgE6dwt1seUX8lCRE0iKRRHEf8ALwBzP7G/AecGtSo5Ly66WXoGlTeOQR+MtfwgBDShAiaVXsqSd3f9LMpgGdAANOdvd5SY9Myp/Fi+HUU6FJExg3DnLSewGcKsKKBIlc9VQfWAu8TLhq6Zdonsj2c4d33w3P69cPN81NmZL2JAEaqU4kTyKd2a8S+icMqAY0AOYDzZIYl5QHixeHsSJefz3UxmjfHtq1K9VdxLYKtlVeK0JlO6S8S+TU00Gx01HZjQuTFpFkv82b4aGH4K9/DS2K++4r9SJ+eQkitsrrtlIrQiRIpEWxFXf/1MzaJiMYKSf+9KfQaTHF0psAABX/SURBVH3sseETfd99S2WzGidCJDmKTRRmdkXMZAXgYGBZ0iKS7LRxI1SoEB6nnQbdu0O/fqVan0mlwEWSI5EWRc2Y5xsJfRbPJSccyUqffQbnnBPujTj//FCCI0nUpyBS+uImiuhGu5ruflWK4pFssm4dDB0Kt98Ou+4Ke+xR6rvQJawiyVdkojCzSu6+0cyOSGVAkiU++QTOOgs+/zz8vOuukCxKQVF9Eep8FkmOeC2KTwj9ETPMbBzwLPBL3kJ3fz7JsUkm+/ln+PVXeOMNOP74Em2iqEtbY5OD+iJEki+RPopqwErCGNl591M4oEQhW3vzTZgzBy6/HI45BubP367yG7Gd07GUHERSK16i+EN0xdNs8hNEHk9qVJJZVq2CK66AUaOgWTO48MKQIEqhRpM6p0XSL16iqAjsyNYJIo8ShQTPPw8XXQQrVsA118CNNyacIIq7a1qd0yJlQ7xE8Y27D0lZJJJ5Fi+G3r3hwAPDgEKtWm3Ty4s6tZRHndMiZUO8RKGR6uX33GHy5NBRUL9+GFyobVuoXLlEm9OpJZGyL1712E4pi0Iyw6JFcMIJ0KFD/qVHRx5Z4iQhIpmhyETh7j+kMhApwzZvhvvvDx3V770H//oXHHVUuqMSkRTZ5qKAUg6dfDK8/HK4H2LYMNhnnxJvSndSi2QeJQop3IYNULFiKOJ3+unQsyf07ZtwEb9EbpZTZ7VIZlCikN/79FPo3z8U8bvwwhIV8dPNciLZQ4lC8v36KwwZAnfcAbvtBnvvvU0vL+y0kq5oEsl8ShQSfPRRKN73xRehJPidd8IuuxT7MhXoE8l+ShQS/PJL6Jd4661QpymOopKDTiuJZCclivLsjTdCEb8rr4ROnUJJ8CpVin2ZRpITKV+UKMqjlStDEb/HHoODDoJLLgkJIoEkkUf9DyLlR7w7syXbuMPYsdC0aWgWXH89TJmyTQlCRMoftSjKk8WLw3mi5s3D2BEtWqQ7IhHJAEoU2c4dJkyAo48Od1RPnAht2kClSsWW+S6K7qgWKV906imbff01HHdc6KiOLk8aPvtwOhxTiQ4d4Lzz8q9a2ha69FWkfFGLIhtt2sQHfe6n1dhr2WwVGdboQV6+8SjcdDmriGw7JYps1L07h7/6KuMrdeHh1g+xolr+HdZKDiKyrZQoMlBhfQsVN29gs1XErQIdv+tLbvXT+fKQPkycpPGnRGT7JLWPwsw6m9l8M1tgZlcXsvwKM5trZjPN7L9mVvL61eVI3g1vefZfPZVhn+bQfdmDAEz4w2l82ebP9PmzkoSIbL+ktSjMrCLwAHAskAtMMbNx7j43ZrXpQI67rzWzC4B/AKclK6Zs0rIlTHz9V7j55lCXaffduezufbisa7ojE5Fsk8wWRRtggbsvdPffgDFA99gV3H2Cu6+NJj8C6iUxnqzS9KcPw30Q//hHKOI3dy50VZYQkdKXzD6KusCSmOlcoG2c9fsDrxe2wMwGAgMB6tevX1rxZYTC+iNmzIBW+/wahih9++1w+auISJKUic5sMzsDyAHaF7bc3YcDwwFycnI8haGlXWwBvrYrX2PftXOg5WCa9Dkazp4HlSunO0QRyXLJTBRLgdiRb+pF87ZiZscA1wHt3X19EuPJGAUHAGrf7HteqncZTHoSWrTggjcHRfWZlCREJPmS2UcxBWhkZg3MrArQGxgXu4KZtQKGAd3c/bskxpJRtlzV5M4Ve41hzMwm8MwzcNNN8MknKuInIimVtBaFu280s4uB8UBF4BF3n2NmQ4Cp7j4OuAPYEXjWzAAWu3u3ZMVU1hRVa2nLMKKPLobGZ4VO65EjQ0lwEZEUS2ofhbu/BrxWYN6NMc/jD6WW5WL7H7Zwp/8+/2X/PseEIn6TJsEhh0DFimmLU0TKtzLRmV2eFOx/2GoAoK++ggEDYOYE2H8i0B4OPTQ9gYqIRJQoUqCoMaa3VGHdtAnuvTcMJFS5MgwbBkcdlbZ4RURiKVGkQLFjTHc5CV5/Pdww9+CDUE/3HYpI2aFEUYqK7ZyeGDPzt9+gUiWoUAH69YO+faF3bzDVZxKRskUDF5WigsX68vxuoJ9PPoHWreHf/w7TvXrB6acrSYhImaQWRSn7Xcsh1tq1cMMNcM89sOeesN9+qQxNRKRElChS5b334KyzYOHCMAbp7bdDrVrpjkpEpFg69VQKhg+HDh0KP+20xYYN4V6ICRPgoYeUJEQkY6hFUQpir2raqi/i5Zdh3jz4y1+gY8dQCrySDrmIZBa1KEpJXt/EwIHAihUhY3TrBqNHhyucQElCRDKSEkVpcg/NiyZNYOxYGDIEPv5YRfxEJKPpK+42KO4+CRYvhrPPhlatQhG/Zs1SHqOISGlToihGUeU38phvZkD9t2jU5/hQxO/dd8M9EiriJyJZQomiEEUlh9+V3/jyy1DEb9YkOGAS0A7atElHyCIiSaNEUYhiazNt3Ah33w033ghVq4bTTCriJyJZSokiErf8d0Fdu8L48dC9eyjDsddeqQpTJKNs2LCB3Nxc1q1bl+5Qyo1q1apRr149KlcuvaGSy3WiKLb8d6z160MJ8AoV4Nxz4Zxz4NRTVZ9JJI7c3Fxq1qzJvvvui+l/JencnZUrV5Kbm0uDBg1KbbvlLlEk3P8Q66OPoH9/OP98uOQS6NkzZfGKZLJ169YpSaSQmVG7dm1WrFhRqtstN4kiL0EknBwAfvklDCZ0771hjIhGjVIWr0i2UJJIrWQc73KTKPI6qItNDnnefTcU8fv6a7jwQrjtNthpp5TEKiJSlmT1ndl5xfryCvZtVWajOBs3hj6JSZPggQeUJEQy2IsvvoiZ8fnnn2+ZN3HiRLp27brVev369WPs2LFA6Ii/+uqradSoEQcffDCHHXYYr7/++nbHctttt9GwYUP2339/xo8fH3fdSy+9lB133HHL9KJFi+jUqRPNmzenQ4cO5Obmbnc8iciKFkVRd0wX20Fd0IsvhiJ+11wTivjNmaP6TCJZYPTo0Rx55JGMHj2aW265JaHX3HDDDXzzzTfMnj2bqlWr8u233zIp70OlhObOncuYMWOYM2cOy5Yt45hjjuGLL76gYiE36E6dOpVVq1ZtNe+qq67izDPP5KyzzuKdd97hmmuu4fHHH9+umBKRFZ+Csfc9xEr4NNO334ZO6mefhYMPhiuvDPWZlCRESs1llxVTir8EWrYM44DFs2bNGt577z0mTJjASSedlFCiWLt2LSNGjODrr7+matWqAOy+++706tVru+J96aWX6N27N1WrVqVBgwY0bNiQTz75hMMOO2yr9TZt2sTgwYN56qmneOGFF7bMnzt3LnfddRcAHTt25OSTT96ueBKVNZ+Ece97KIo7PPFE+Ateswb+9jcYPDicchKRrPDSSy/RuXNnGjduTO3atZk2bRqtW7eO+5oFCxZQv359dkrglPPll1/OhAkTfje/d+/eXH311VvNW7p0KYceeuiW6Xr16rF06dLfvfb++++nW7du7LnnnlvNb9GiBc8//zyDBg3ihRdeYPXq1axcuZLatWsXG+f2yJpEUSKLF4d7InJywt3VBxyQ7ohEslZx3/yTZfTo0QwaNAgIH96jR4+mdevWRV4dtK1XDd19993bHWOsZcuW8eyzzzKxkG++d955JxdffDGjRo2iXbt21K1bt9DTVqWt/CWKzZvDXdUnnBCK+L3/fqj2qiJ+Ilnnhx9+4J133mHWrFmYGZs2bcLMuOOOO6hdu/bv+gB++OEH6tSpQ8OGDVm8eDE///xzsa2KbWlR1K1blyVLlmyZzs3NpW7dulutM336dBYsWEDDhg2BcBqsYcOGLFiwgL322ovnn38eCKfUnnvuOXbeeefED0hJuXtGPVq3bu3u7sOGubdvHx61aoWfxZo/3/2oo9zBfeLEBF4gIttj7ty5ad3/sGHDfODAgVvNa9eunU+aNMnXrVvn++6775YY//e//3n9+vX9xx9/dHf3wYMHe79+/Xz9+vXu7v7dd9/5M888s13xzJ4925s3b+7r1q3zhQsXeoMGDXzjxo1xX1OjRo0tz1esWOGbNm1yd/drr73Wb7jhhkJfU9hxB6Z6CT93M/by2LwObEjgiqaNG+H226F5c5g1C/7zH2jXLiVxikj6jB49mlNOOWWreT169GD06NFUrVqVJ554grPPPpuWLVvSs2dPHn74YWpF49kPHTqU3XbbjaZNm3LggQfStWvXhPos4mnWrBm9evWiadOmdO7cmQceeGDLqaMuXbqwbNmyuK+fOHEi+++/P40bN+bbb7/luuuu2654EmUh0WSOnJwcnzp1Kh06hOmEOrCPPx7efBP+9KdwT8QeeyQxQhHJM2/ePJo0aZLuMMqdwo67mU1z95ySbC97+yjWrQtXL1WsGK6PHTgQevRId1QiIhkn4049zZ+ff6d1kd5/P5yPeuCBMN2jh5KEiEgJZVyi+PXX8LPQfok1a+DSS8MgQuvWgZq8ImmXaae3M10yjnfGnXqqXr2IfolJk0IRv8WL4eKL4dZbIaZGioikXrVq1bbcEKYqssnn0XgU1apVK9XtZlyiiGuHHULV1yOOSHckIkK48zg3N7fUx0eQouWNcFeaMu6qp5o1c3z16qlh4vnn4fPP4dprw/SmTbpxTkSkENtz1VNS+yjMrLOZzTezBWZ2dSHLq5rZ09Hyj81s34Q2vHx5GGWuRw944QX47bcwX0lCRKTUJS1RmFlF4AHgBKApcLqZNS2wWn9glbs3BO4Gbi9uu7U2rAyd1K+8EgYT+uCDUOlVRESSIpktijbAAndf6O6/AWOA7gXW6Q48Gj0fC3SyYnq8dl+/CA48ED77DK6+WpVeRUSSLJmd2XWBJTHTuUDbotZx941m9hNQG/g+diUzGwjkjSqx3t57b7YqvQJQhwLHqhzTscinY5FPxyLf/iV9YUZc9eTuw4HhAGY2taQdMtlGxyKfjkU+HYt8Ohb5zGxqSV+bzFNPS4G9Y6brRfMKXcfMKgG1gJVJjElERLZRMhPFFKCRmTUwsypAb2BcgXXGAWdFz3sC73imXa8rIpLlknbqKepzuBgYD1QEHnH3OWY2hFAXfRwwEnjczBYAPxCSSXGGJyvmDKRjkU/HIp+ORT4di3wlPhYZd8OdiIikVsYVBRQRkdRSohARkbjKbKJIWvmPDJTAsbjCzOaa2Uwz+6+Z7ZOOOFOhuGMRs14PM3Mzy9pLIxM5FmbWK/rbmGNmT6U6xlRJ4H+kvplNMLPp0f9Jl3TEmWxm9oiZfWdms4tYbmZ2X3ScZprZwQltuKSDbSfzQej8/gr4I1AF+AxoWmCdC4GHoue9gafTHXcaj0VHYIfo+QXl+VhE69UEJgMfATnpjjuNfxeNgOnALtH0H9IddxqPxXDgguh5U+B/6Y47SceiHXAwMLuI5V2A1wEDDgU+TmS7ZbVFkZTyHxmq2GPh7hPcfW00+RHhnpVslMjfBcD/EeqGrUtlcCmWyLEYADzg7qsA3P27FMeYKokcCwd2ip7XApalML6UcffJhCtIi9IdeMyDj4CdzWzP4rZbVhNFYeU/6ha1jrtvBPLKf2SbRI5FrP6EbwzZqNhjETWl93b3V1MZWBok8nfRGGhsZu+b2Udm1jll0aVWIsfiZuAMM8sFXgMuSU1oZc62fp4AGVLCQxJjZmcAOUD7dMeSDmZWAbgL6JfmUMqKSoTTTx0IrczJZnaQu/+Y1qjS43RglLv/08wOI9y/daC7b053YJmgrLYoVP4jXyLHAjM7BrgO6Obu61MUW6oVdyxqAgcCE83sf4RzsOOytEM7kb+LXGCcu29w96+BLwiJI9skciz6A88AuPuHQDVCwcDyJqHPk4LKaqJQ+Y98xR4LM2sFDCMkiWw9Dw3FHAt3/8nd67j7vu6+L6G/ppu7l7gYWhmWyP/Ii4TWBGZWh3AqamEqg0yRRI7FYqATgJk1ISSK8jg+6zjgzOjqp0OBn9z9m+JeVCZPPXnyyn9knASPxR3AjsCzUX/+YnfvlragkyTBY1EuJHgsxgPHmdlcYBMw2N2zrtWd4LG4EhhhZpcTOrb7ZeMXSzMbTfhyUCfqj7kJqAzg7g8R+me6AAuAtcDZCW03C4+ViIiUorJ66klERMoIJQoREYlLiUJEROJSohARkbiUKEREJC4lCimTzGyTmc2IeewbZ901pbC/UWb2dbSvT6O7d7d1Gw+bWdPo+bUFln2wvTFG28k7LrPN7GUz27mY9Vtma6VUSR1dHitlkpmtcfcdS3vdONsYBbzi7mPN7DjgTndvvh3b2+6YituumT0KfOHuf4uzfj9CBd2LSzsWKT/UopCMYGY7RmNtfGpms8zsd1VjzWxPM5sc8437qGj+cWb2YfTaZ82suA/wyUDD6LVXRNuabWaXRfNqmNmrZvZZNP+0aP5EM8sxs78D1aM4noyWrYl+jjGzE2NiHmVmPc2sopndYWZTonECzkvgsHxIVNDNzNpE73G6mX1gZvtHdykPAU6LYjktiv0RM/skWrew6rsiW0t3/XQ99CjsQbiTeEb0eIFQRWCnaFkdwp2leS3iNdHPK4HroucVCbWf6hA++GtE8/8K3FjI/kYBPaPnpwIfA62BWUANwp3vc4BWQA9gRMxra0U/JxKNf5EXU8w6eTGeAjwaPa9CqORZHRgIXB/NrwpMBRoUEueamPf3LNA5mt4JqBQ9PwZ4LnreD7g/5vW3AmdEz3cm1H+qke7ftx5l+1EmS3iIAL+6e8u8CTOrDNxqZu2AzYRv0rsDy2NeMwV4JFr3RXefYWbtCQPVvB+VN6lC+CZemDvM7HpCDaD+hNpAL7j7L1EMzwNHAW8A/zSz2wmnq97dhvf1OnCvmVUFOgOT3f3X6HRXczPrGa1Xi1DA7+sCr69uZjOi9z8PeCtm/UfNrBGhREXlIvZ/HNDNzK6KpqsB9aNtiRRKiUIyxZ+B3YDW7r7BQnXYarEruPvkKJGcCIwys7uAVcBb7n56AvsY7O5j8ybMrFNhK7n7FxbGvegCDDWz/7r7kETehLuvM7OJwPHAaYRBdiCMOHaJu48vZhO/untLM9uBUNvoIuA+wmBNE9z9lKjjf2IRrzegh7vPTyReEVAfhWSOWsB3UZLoCPxuXHALY4V/6+4jgIcJQ0J+BBxhZnl9DjXMrHGC+3wXONnMdjCzGoTTRu+a2V7AWnd/glCQsbBxhzdELZvCPE0oxpbXOoHwoX9B3mvMrHG0z0J5GNHwUuBKyy+zn1cuul/MqqsJp+DyjAcusah5ZaHysEhcShSSKZ4EcsxsFnAm8Hkh63QAPjOz6YRv6/e6+wrCB+doM5tJOO10QCI7dPdPCX0XnxD6LB529+nAQcAn0Smgm4Chhbx8ODAzrzO7gDcJg0u97WHoTgiJbS7wqZnNJpSNj9vij2KZSRiU5x/AbdF7j33dBKBpXmc2oeVROYptTjQtEpcujxURkbjUohARkbiUKEREJC4lChERiUuJQkRE4lKiEBGRuJQoREQkLiUKERGJ6/8BpkqKcmHKfaMAAAAASUVORK5CYII=\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "output_type": "unknown"
        }
      ],
      "source": [
        "# Compute predicted probabilities on the test set\n",
        "probs = bert_predict(bert_classifier, val_dataloader)\n",
        "\n",
        "# Evaluate the Bert classifier\n",
        "evaluate_roc(probs, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vxHGJj7BfhWv",
        "outputId": "cf1e94c2-8739-4508-8805-60781b2c3b7a"
      },
      "outputs": [
        {
          "data": {
            "application/x.notebook.stream": "Tokenizing data...\nNumber of tweets predicted as Rumor:  391\n\n               precision    recall  f1-score   support\n\n           0       0.22      0.18      0.20       116\n           1       0.76      0.80      0.78       369\n\n    accuracy                           0.65       485\n   macro avg       0.49      0.49      0.49       485\nweighted avg       0.63      0.65      0.64       485\n\n0.6536082474226804\n0.7789473684210527\n"
          },
          "output_type": "unknown"
        }
      ],
      "source": [
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_val, y_val)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "yu1vQFK_fhWv",
        "outputId": "05e4aaa2-74fa-4254-a47b-6218057d8e92"
      },
      "outputs": [
        {
          "data": {
            "application/x.notebook.stream": "Tokenizing data...\nNumber of tweets predicted as Rumor:  3823\n\n               precision    recall  f1-score   support\n\n           0       0.11      0.33      0.17       475\n           1       0.92      0.74      0.82      4752\n\n    accuracy                           0.70      5227\n   macro avg       0.52      0.54      0.49      5227\nweighted avg       0.84      0.70      0.76      5227\n\n0.7013583317390473\n0.8179591836734694\n"
          },
          "output_type": "unknown"
        }
      ],
      "source": [
        "rhi_data = pd.read_csv('data/_RHI_text.csv')\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv')\n",
        "X_test = rhi_data.text.values\n",
        "y_test = rhi_y.isRumor.values\n",
        "\n",
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))\n",
        "testing_process(bert_classifier, X_test, y_test)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "F08yfWtRAzty"
      },
      "source": [
        "## BERTweet Embedding Only"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {},
      "outputs": [],
      "source": [
        "device = getDevice()\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
        "bert_classifier = BertTweetClassifier(freeze_bert=True)\n",
        "\n",
        "PATH = './Model/BERTweet_raw_embedding.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {},
      "source": [
        "#### BERT & EMOJI 이모지 다루는 차이 😂😂😂😂😂\n",
        "\n",
        "> Before applying fastBPE to the pre-training corpus of 850M English Tweets, we tokenized these Tweets using TweetTokenizer from the NLTK toolkit and used the emoji package to translate emotion icons into text strings (here, each icon is referred to as a word token). We also normalized the Tweets by converting user mentions and web/url links into special tokens @USER and HTTPURL, respectively. Thus it is recommended to also apply the same pre-processing step for BERTweet-based downstream applications w.r.t. the raw input Tweets. BERTweet provides this pre-processing step by enabling the normalization argument.\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 31,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "<Normal text>\n",
            "https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂 http://www.google.com \n",
            "\n",
            "['https://www.naver.com', '@MichaelEssien', \"that's\", 'a', 'shame', 'wanted', 'to', 'Invest', 'INVEST', 'in', 'you😔😂😂', 'http://www.google.com']\n",
            "\n",
            " [0, 62060, 8798, 3, 36110, 1626, 2237, 6354, 5238, 55508, 471, 10424, 818, 6139, 20, 11, 2536, 588, 9, 22630, 27227, 3969, 16, 3805, 3, 3, 3, 45565, 36110, 9485, 17048, 6354, 2] \n",
            "\n",
            "--------------------------------------------------\n",
            "<Demojized text>\n",
            "https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy: http://www.google.com \n",
            "\n",
            "['https://www.naver.com', '@MichaelEssien', \"that's\", 'a', 'shame', 'wanted', 'to', 'Invest', 'INVEST', 'in', 'you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy:', 'http://www.google.com']\n",
            "\n",
            " [0, 62060, 8798, 3, 36110, 1626, 2237, 6354, 5238, 55508, 471, 10424, 818, 6139, 20, 11, 2536, 588, 9, 22630, 27227, 3969, 16, 3805, 3, 524, 9859, 1043, 3, 3, 16517, 1043, 88, 45565, 36110, 9485, 17048, 6354, 2] \n",
            "\n",
            "--------------------------------------------------\n",
            "<Demojized & http text>\n",
            "HTTPURL @MichaelEssien that's a shame wanted to Invest INVEST in you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy: HTTPURL \n",
            "\n",
            "['HTTPURL', '@MichaelEssien', \"that's\", 'a', 'shame', 'wanted', 'to', 'Invest', 'INVEST', 'in', 'you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy:', 'HTTPURL']\n",
            "\n",
            " [0, 10, 5238, 55508, 471, 10424, 818, 6139, 20, 11, 2536, 588, 9, 22630, 27227, 3969, 16, 3805, 3, 524, 9859, 1043, 3, 3, 16517, 1043, 88, 10, 2] \n",
            "\n",
            "--------------------------------------------------\n",
            "<Demojized & http & Mention text>\n",
            "HTTPURL @USER that's a shame wanted to Invest INVEST in you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy: HTTPURL \n",
            "\n",
            "['HTTPURL', '@USER', \"that's\", 'a', 'shame', 'wanted', 'to', 'Invest', 'INVEST', 'in', 'you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy:', 'HTTPURL']\n",
            "\n",
            " [0, 10, 5, 6139, 20, 11, 2536, 588, 9, 22630, 27227, 3969, 16, 3805, 3, 524, 9859, 1043, 3, 3, 16517, 1043, 88, 10, 2] \n",
            "\n",
            "--------------------------------------------------\n",
            "<Demojized-spaced & http & Mention text>\n",
            "HTTPURL @USER that's a shame wanted to Invest INVEST in you :pensive_face: :face_with_tears_of_joy: :face_with_tears_of_joy: HTTPURL \n",
            "\n",
            "['HTTPURL', '@USER', \"that's\", 'a', 'shame', 'wanted', 'to', 'Invest', 'INVEST', 'in', 'you', ':pensive_face:', ':face_with_tears_of_joy:', ':face_with_tears_of_joy:', 'HTTPURL']\n",
            "\n",
            " [0, 10, 5, 6139, 20, 11, 2536, 588, 9, 22630, 27227, 3969, 16, 14, 908, 88, 88, 10, 2] \n",
            "\n"
          ]
        }
      ],
      "source": [
        "# INPUT TWEET IS ALREADY NORMALIZED!\n",
        "# print(\"<Normal text>\")\n",
        "# line = \"@MichaelEssien that's a shame wanted to Invest in you😔😂😂\"\n",
        "# print(line)\n",
        "# print(tokenizer.encode(line),\"\\n\")\n",
        "# input_ids = torch.tensor([tokenizer.encode(line)])\n",
        "# # print(input_ids)\n",
        "\n",
        "print(\"<Normal text>\")\n",
        "line = \"https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂 http://www.google.com\"\n",
        "input_ids = torch.tensor([tokenizer.encode(line)])\n",
        "print(line,\"\\n\")\n",
        "print(line.split())\n",
        "print(\"\\n\",tokenizer.encode(line),\"\\n\")\n",
        "print(\"--------------------------------------------------\")\n",
        "\n",
        "print(\"<Demojized text>\")\n",
        "# line = \"HTTPURL @USER that's a shame wanted to Invest INVEST in you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy: HTTPURL\"\n",
        "line = \"https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂 http://www.google.com\"\n",
        "line = emoji.demojize(line)\n",
        "print(line,\"\\n\")\n",
        "print(line.split())\n",
        "print(\"\\n\",tokenizer.encode(line),\"\\n\")\n",
        "print(\"--------------------------------------------------\")\n",
        "\n",
        "print(\"<Demojized & http text>\")\n",
        "# line = \"HTTPURL @USER that's a shame wanted to Invest INVEST in you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy: HTTPURL\"\n",
        "line = \"https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂 http://www.google.com\"\n",
        "line = emoji.demojize(line)\n",
        "line = re.sub(r\"http\\S+\", \"HTTPURL\", line)  # http link -> '*'\n",
        "print(line,\"\\n\")\n",
        "print(line.split())\n",
        "print(\"\\n\",tokenizer.encode(line),\"\\n\")\n",
        "print(\"--------------------------------------------------\")\n",
        "\n",
        "print(\"<Demojized & http & Mention text>\")\n",
        "# line = \"HTTPURL @USER that's a shame wanted to Invest INVEST in you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy: HTTPURL\"\n",
        "line = \"https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂 http://www.google.com\"\n",
        "line = emoji.demojize(line)\n",
        "line = re.sub(r\"http\\S+\", \"HTTPURL\", line)  # http link -> '*'\n",
        "line = re.sub(r\"@\\S+\", \"@USER\", line)   # mention -> '@'\n",
        "print(line,\"\\n\")\n",
        "print(line.split())\n",
        "input_ids = torch.tensor([tokenizer.encode(line)])\n",
        "print(\"\\n\",tokenizer.encode(line),\"\\n\")\n",
        "print(\"--------------------------------------------------\")\n",
        "\n",
        "print(\"<Demojized-spaced & http & Mention text>\")\n",
        "# line = \"HTTPURL @USER that's a shame wanted to Invest INVEST in you:pensive_face::face_with_tears_of_joy::face_with_tears_of_joy: HTTPURL\"\n",
        "line = \"https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂 http://www.google.com\"\n",
        "line = emoji.demojize(line)\n",
        "line = re.sub(r\"http\\S+\", \"HTTPURL\", line)  # http link -> '*'\n",
        "line = re.sub(r\"@\\S+\", \"@USER\", line)   # mention -> '@'\n",
        "line = re.sub(r':[^:]*:', r' \\g<0>', line)  # http link -> '*'\n",
        "input_ids = torch.tensor([tokenizer.encode(line)])\n",
        "print(line,\"\\n\")\n",
        "print(line.split())\n",
        "print(\"\\n\",tokenizer.encode(line),\"\\n\")\n",
        "\n",
        "# line = \"HTTPURL @USER that's a shame wanted to Invest INVEST in you pensive_face face_with_tears_of_joy face_with_tears_of_joy :grinning_face_with_big_eyes: HTTPURL  \"\n",
        "# input_ids = torch.tensor([tokenizer.encode(line)])\n",
        "# print(line)\n",
        "# print(tokenizer.encode(line))\n",
        "\n",
        "# with torch.no_grad():\n",
        "#     features = bertweet(input_ids)  # Models outputs are now tuples\n",
        "# print(features)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 30,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[':pensive_face:', ':face_with_tears_of_joy:']\n",
            "tensor([[  0, 908,  88,   2]])\n"
          ]
        }
      ],
      "source": [
        "line = \":pensive_face: :face_with_tears_of_joy:\"\n",
        "print(line.split())\n",
        "input_ids = torch.tensor([tokenizer.encode(line)])\n",
        "print(input_ids)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {},
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "4 \n",
            "\n",
            "HTTPURL @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂😃 http://www.google.com\n",
            "['::', '::', '::']\n",
            "HTTPURL @USER that's a shame wanted to Invest INVEST in you :pensive_face: :face_with_tears_of_joy: :face_with_tears_of_joy: :grinning_face_with_big_eyes: HTTPURL \n",
            "\n",
            "['HTTPURL', '@USER', \"that's\", 'a', 'shame', 'wanted', 'to', 'Invest', 'INVEST', 'in', 'you', ':pensive_face:', ':face_with_tears_of_joy:', ':face_with_tears_of_joy:', ':grinning_face_with_big_eyes:', 'HTTPURL'] \n",
            "\n"
          ]
        }
      ],
      "source": [
        "text = \"HTTPURL @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂😃 http://www.google.com\"\n",
        "print(emoji.emoji_count(text),\"\\n\")\n",
        "print(text)\n",
        "text = emoji.demojize(text)\n",
        "# print(emoji.get_emoji_regexp(),\"\\n\")\n",
        "# text=text.strip(':')\n",
        "# text = re.sub(r'(@.*?)[\\s]', '@USER ', text)\n",
        "text = re.sub(r\"@\\S+\", \"@USER\", text)   # mention -> '@'\n",
        "text = re.sub(r\"http\\S+\", \"HTTPURL\", text)  # http link -> '*'\n",
        "emojis = re.findall(r'(::)', text)\n",
        "print(emojis)\n",
        "# print(text,\"\\n\")\n",
        "text = re.sub(r':[^:]*:', r' \\g<0>', text)  # http link -> '*'\n",
        "print(text,\"\\n\")\n",
        "\n",
        "text=text.split()\n",
        "# text= re.sub(r'(:[!_\\-\\w]+:)', '', text)\n",
        "print(text,\"\\n\")\n",
        "# emojis"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "JGR4vVZKqbhf"
      },
      "outputs": [],
      "source": [
        "def getEmbeddingBERT(bert_classifier, data):\n",
        "  bert_classifier.eval()\n",
        "  input_ids = []\n",
        "  attention_masks = []\n",
        "  result = []\n",
        "\n",
        "  # For every sentence...\n",
        "  for index, sent in enumerate(data):\n",
        "      encoded_sent = tokenizer.encode(\n",
        "          text=text_preprocessing(sent),  # Preprocess sentence\n",
        "          add_special_tokens=True,        # Add `[CLS]` and `[SEP]`\n",
        "          return_attention_mask=False,      # Return attention mask\n",
        "\n",
        "          padding='do_not_pad'\n",
        "      )\n",
        "      with torch.no_grad():\n",
        "          encoded_sent = torch.tensor([encoded_sent])\n",
        "          # print(encoded_sent)\n",
        "          last_hidden_state = bert_classifier.embedding(encoded_sent)\n",
        "\n",
        "      result.append(last_hidden_state)\n",
        "\n",
        "      if (index % 300 == 0 or index == len(input_ids)): print(\"Current number of processed tweets: {}\".format(index))\n",
        "  return result\n",
        "\n",
        "\n",
        "line = \"https://www.naver.com @MichaelEssien that's a shame wanted to Invest INVEST in you😔😂😂 http://www.google.com\"\n",
        "line = emoji.demojize(line)\n",
        "line = re.sub(r\"http\\S+\", \"HTTPURL\", line)  # http link -> '*'\n",
        "line = re.sub(r\"@\\S+\", \"@USER\", line)   # mention -> '@'\n",
        "line = re.sub(r':[^:]*:', r' \\g<0>', line)  # http link -> '*'\n",
        "input_ids = torch.tensor([tokenizer.encode(line)])\n",
        "print(line,\"\\n\")\n",
        "print(line.split())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Q1hpuq6NqbJ_"
      },
      "outputs": [],
      "source": [
        "raw_text = pd.read_csv('./data/_PHEME_text.csv')\n",
        "y = pd.read_csv('./data/_PHEME_target.csv')\n",
        "data = pd.concat([raw_text.text, y], axis=1).reset_index(drop=True)\n",
        "val = pd.read_csv('data/_PHEMEext_text.csv')\n",
        "\n",
        "X_train = data.text.values\n",
        "y_train = data.target.values\n",
        "\n",
        "X_val = val.drop(['Event'],axis=1).text.values\n",
        "y_val = val.target.values\n",
        "\n",
        "rhi_data = pd.read_csv('data/_RHI_text.csv').text.values\n",
        "rhi_y = pd.read_csv('data/_RHI_text.csv').isRumor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 275,
          "referenced_widgets": [
            "2c22c266bd5c4ccea6b245e4b990971c",
            "be000548455044719249d519495e514a",
            "7f62faaa6cf947f48cde0b97d5fff4d7",
            "fb8c67a56d8b4c4986c6d815cb4c6b3e",
            "9eb0aaa6736e43c58cc44ce1504e5366",
            "dd7dc5791f9142138fa0265c49d4be74",
            "dcc1de2c1c1948e98c097472cea17686",
            "283c37bb7a4c42c2853f77a0e82c9a0e",
            "be973816203644c69f9ba660c08b373c",
            "b19ffd9730f54aa2aa484ba1e6e418d9",
            "c4a9e8b9afe5416b8e229a4f078dc238",
            "68ddd0f509094751995c794a0dd9d3b0",
            "0a75ab30a15140db902caf400674c115",
            "acbcd81cab3b42539ac733b7c19098c1",
            "6b12e2ebcfcf4088a81abd6839a1fa06",
            "b0e7b89c3e314dec92e6d355c5b8f99e",
            "5ae373acd0d6406782cdd393667a5eae",
            "0d25491d17b242b38c658b3ad668de1e",
            "e956c2c49a7648e1911ff94c3c403c9b",
            "ab141a8f69d54292abcd5fb01104fdd0",
            "08b32242d368498daa7828b7afeb2a9d",
            "cdf8aa309771498786713193c162b7c5",
            "1a43d70de32d4def9a1fe6966d96650b",
            "6f2c1baae7c7462091f69527d989a792",
            "d4ed58d884a84191a2837daa47cba485",
            "3b53ece024be4582879ecfb989e30b75",
            "1e80b1e80add4325a6750caeab4c54b6",
            "d82470e0c93e40bba3a66cbe425393bb",
            "b19a9345a08d4154aaa3aec5d34b7a13",
            "0dcbf81848f9482ea0bcb415241d13bb",
            "9e30225cf2364a548c06b6eecc8727df",
            "c070a13922844983ad5ae16730544deb"
          ]
        },
        "id": "FKpHlV3LoRqG",
        "outputId": "34b5d876-d3b0-4604-ac96-7c22a052dac9"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "There are 1 GPU(s) available.\n",
            "Device name: Tesla P4\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "2c22c266bd5c4ccea6b245e4b990971c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=558.0, style=ProgressStyle(description_…"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "be973816203644c69f9ba660c08b373c",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=843438.0, style=ProgressStyle(descripti…"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "5ae373acd0d6406782cdd393667a5eae",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=1078931.0, style=ProgressStyle(descript…"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Special tokens have been added in the vocabulary, make sure the associated word embedding are fine-tuned or trained.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "d4ed58d884a84191a2837daa47cba485",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=542529064.0, style=ProgressStyle(descri…"
            ]
          },
          "metadata": {
            "tags": []
          },
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<All keys matched successfully>"
            ]
          },
          "execution_count": 11,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "device = getDevice()\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"vinai/bertweet-base\", use_fast=False)\n",
        "bert_classifier = BertTweetClassifier(freeze_bert=True)\n",
        "\n",
        "PATH = './Model/BERTweet_raw_to_fine_tune_ord4.pt'\n",
        "bert_classifier.load_state_dict(torch.load(PATH))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "q6GMtJxgE0yX",
        "outputId": "507d3ab0-f445-4468-902a-5bb738b1ac5e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Current number of processed tweets: 0\n",
            "Current number of processed tweets: 300\n"
          ]
        }
      ],
      "source": [
        "result = getEmbeddingBERT(bert_classifier,X_val)\n",
        "df_result = pd.DataFrame([sent[0].tolist() for sent in result]).add_prefix('BERTEmbed_')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "D-fILEMUf4rm",
        "outputId": "c43507ba-3e21-4b90-f3b2-9b082c5fe4b4"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>BERTEmbed_0</th>\n",
              "      <th>BERTEmbed_1</th>\n",
              "      <th>BERTEmbed_2</th>\n",
              "      <th>BERTEmbed_3</th>\n",
              "      <th>BERTEmbed_4</th>\n",
              "      <th>BERTEmbed_5</th>\n",
              "      <th>BERTEmbed_6</th>\n",
              "      <th>BERTEmbed_7</th>\n",
              "      <th>BERTEmbed_8</th>\n",
              "      <th>BERTEmbed_9</th>\n",
              "      <th>BERTEmbed_10</th>\n",
              "      <th>BERTEmbed_11</th>\n",
              "      <th>BERTEmbed_12</th>\n",
              "      <th>BERTEmbed_13</th>\n",
              "      <th>BERTEmbed_14</th>\n",
              "      <th>BERTEmbed_15</th>\n",
              "      <th>BERTEmbed_16</th>\n",
              "      <th>BERTEmbed_17</th>\n",
              "      <th>BERTEmbed_18</th>\n",
              "      <th>BERTEmbed_19</th>\n",
              "      <th>BERTEmbed_20</th>\n",
              "      <th>BERTEmbed_21</th>\n",
              "      <th>BERTEmbed_22</th>\n",
              "      <th>BERTEmbed_23</th>\n",
              "      <th>BERTEmbed_24</th>\n",
              "      <th>BERTEmbed_25</th>\n",
              "      <th>BERTEmbed_26</th>\n",
              "      <th>BERTEmbed_27</th>\n",
              "      <th>BERTEmbed_28</th>\n",
              "      <th>BERTEmbed_29</th>\n",
              "      <th>BERTEmbed_30</th>\n",
              "      <th>BERTEmbed_31</th>\n",
              "      <th>BERTEmbed_32</th>\n",
              "      <th>BERTEmbed_33</th>\n",
              "      <th>BERTEmbed_34</th>\n",
              "      <th>BERTEmbed_35</th>\n",
              "      <th>BERTEmbed_36</th>\n",
              "      <th>BERTEmbed_37</th>\n",
              "      <th>BERTEmbed_38</th>\n",
              "      <th>BERTEmbed_39</th>\n",
              "      <th>...</th>\n",
              "      <th>BERTEmbed_728</th>\n",
              "      <th>BERTEmbed_729</th>\n",
              "      <th>BERTEmbed_730</th>\n",
              "      <th>BERTEmbed_731</th>\n",
              "      <th>BERTEmbed_732</th>\n",
              "      <th>BERTEmbed_733</th>\n",
              "      <th>BERTEmbed_734</th>\n",
              "      <th>BERTEmbed_735</th>\n",
              "      <th>BERTEmbed_736</th>\n",
              "      <th>BERTEmbed_737</th>\n",
              "      <th>BERTEmbed_738</th>\n",
              "      <th>BERTEmbed_739</th>\n",
              "      <th>BERTEmbed_740</th>\n",
              "      <th>BERTEmbed_741</th>\n",
              "      <th>BERTEmbed_742</th>\n",
              "      <th>BERTEmbed_743</th>\n",
              "      <th>BERTEmbed_744</th>\n",
              "      <th>BERTEmbed_745</th>\n",
              "      <th>BERTEmbed_746</th>\n",
              "      <th>BERTEmbed_747</th>\n",
              "      <th>BERTEmbed_748</th>\n",
              "      <th>BERTEmbed_749</th>\n",
              "      <th>BERTEmbed_750</th>\n",
              "      <th>BERTEmbed_751</th>\n",
              "      <th>BERTEmbed_752</th>\n",
              "      <th>BERTEmbed_753</th>\n",
              "      <th>BERTEmbed_754</th>\n",
              "      <th>BERTEmbed_755</th>\n",
              "      <th>BERTEmbed_756</th>\n",
              "      <th>BERTEmbed_757</th>\n",
              "      <th>BERTEmbed_758</th>\n",
              "      <th>BERTEmbed_759</th>\n",
              "      <th>BERTEmbed_760</th>\n",
              "      <th>BERTEmbed_761</th>\n",
              "      <th>BERTEmbed_762</th>\n",
              "      <th>BERTEmbed_763</th>\n",
              "      <th>BERTEmbed_764</th>\n",
              "      <th>BERTEmbed_765</th>\n",
              "      <th>BERTEmbed_766</th>\n",
              "      <th>BERTEmbed_767</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.301987</td>\n",
              "      <td>0.213715</td>\n",
              "      <td>-0.498807</td>\n",
              "      <td>0.209999</td>\n",
              "      <td>0.195544</td>\n",
              "      <td>0.207029</td>\n",
              "      <td>0.191929</td>\n",
              "      <td>-0.076861</td>\n",
              "      <td>0.281421</td>\n",
              "      <td>0.275451</td>\n",
              "      <td>0.080227</td>\n",
              "      <td>-0.070921</td>\n",
              "      <td>0.134354</td>\n",
              "      <td>-0.329015</td>\n",
              "      <td>-0.031797</td>\n",
              "      <td>0.542257</td>\n",
              "      <td>0.087555</td>\n",
              "      <td>0.243908</td>\n",
              "      <td>-0.179307</td>\n",
              "      <td>0.443024</td>\n",
              "      <td>-0.359229</td>\n",
              "      <td>-0.137068</td>\n",
              "      <td>0.354704</td>\n",
              "      <td>0.124060</td>\n",
              "      <td>0.093317</td>\n",
              "      <td>-0.171579</td>\n",
              "      <td>-0.077139</td>\n",
              "      <td>0.099242</td>\n",
              "      <td>0.061917</td>\n",
              "      <td>0.538866</td>\n",
              "      <td>0.074907</td>\n",
              "      <td>-0.131123</td>\n",
              "      <td>0.273107</td>\n",
              "      <td>-0.306408</td>\n",
              "      <td>0.163046</td>\n",
              "      <td>0.085462</td>\n",
              "      <td>0.209216</td>\n",
              "      <td>0.359416</td>\n",
              "      <td>0.443941</td>\n",
              "      <td>0.596167</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.375472</td>\n",
              "      <td>0.238702</td>\n",
              "      <td>0.162483</td>\n",
              "      <td>-0.218867</td>\n",
              "      <td>-0.102786</td>\n",
              "      <td>-0.171557</td>\n",
              "      <td>0.120789</td>\n",
              "      <td>-0.160129</td>\n",
              "      <td>0.061481</td>\n",
              "      <td>-0.188779</td>\n",
              "      <td>-0.321238</td>\n",
              "      <td>0.034923</td>\n",
              "      <td>-0.017988</td>\n",
              "      <td>-0.164916</td>\n",
              "      <td>-0.296818</td>\n",
              "      <td>0.222189</td>\n",
              "      <td>-0.001456</td>\n",
              "      <td>0.303296</td>\n",
              "      <td>-0.208206</td>\n",
              "      <td>0.195024</td>\n",
              "      <td>0.575484</td>\n",
              "      <td>-0.085796</td>\n",
              "      <td>0.226221</td>\n",
              "      <td>0.111809</td>\n",
              "      <td>0.410422</td>\n",
              "      <td>-0.048327</td>\n",
              "      <td>-0.355674</td>\n",
              "      <td>0.041619</td>\n",
              "      <td>0.376790</td>\n",
              "      <td>0.295959</td>\n",
              "      <td>0.240020</td>\n",
              "      <td>-0.169549</td>\n",
              "      <td>0.251581</td>\n",
              "      <td>-0.580530</td>\n",
              "      <td>-0.131731</td>\n",
              "      <td>0.219126</td>\n",
              "      <td>-0.132782</td>\n",
              "      <td>0.038242</td>\n",
              "      <td>-0.040448</td>\n",
              "      <td>-0.073326</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.248635</td>\n",
              "      <td>0.184559</td>\n",
              "      <td>-0.495903</td>\n",
              "      <td>0.237322</td>\n",
              "      <td>0.221858</td>\n",
              "      <td>0.227700</td>\n",
              "      <td>0.107580</td>\n",
              "      <td>0.022934</td>\n",
              "      <td>0.406658</td>\n",
              "      <td>0.248065</td>\n",
              "      <td>0.120567</td>\n",
              "      <td>-0.019171</td>\n",
              "      <td>0.032085</td>\n",
              "      <td>-0.226372</td>\n",
              "      <td>0.093066</td>\n",
              "      <td>0.573977</td>\n",
              "      <td>0.145331</td>\n",
              "      <td>0.347341</td>\n",
              "      <td>-0.165440</td>\n",
              "      <td>0.443770</td>\n",
              "      <td>-0.404685</td>\n",
              "      <td>-0.206766</td>\n",
              "      <td>0.446454</td>\n",
              "      <td>0.146747</td>\n",
              "      <td>0.042815</td>\n",
              "      <td>-0.180195</td>\n",
              "      <td>-0.044686</td>\n",
              "      <td>0.094516</td>\n",
              "      <td>0.146544</td>\n",
              "      <td>0.464722</td>\n",
              "      <td>0.054440</td>\n",
              "      <td>-0.073036</td>\n",
              "      <td>0.214099</td>\n",
              "      <td>-0.381920</td>\n",
              "      <td>0.165374</td>\n",
              "      <td>0.085333</td>\n",
              "      <td>0.193061</td>\n",
              "      <td>0.427717</td>\n",
              "      <td>0.399479</td>\n",
              "      <td>0.623183</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.412934</td>\n",
              "      <td>0.246606</td>\n",
              "      <td>0.160106</td>\n",
              "      <td>-0.164791</td>\n",
              "      <td>-0.130812</td>\n",
              "      <td>-0.268747</td>\n",
              "      <td>0.126012</td>\n",
              "      <td>-0.204678</td>\n",
              "      <td>0.126123</td>\n",
              "      <td>-0.157855</td>\n",
              "      <td>-0.267253</td>\n",
              "      <td>0.025122</td>\n",
              "      <td>-0.029456</td>\n",
              "      <td>-0.162739</td>\n",
              "      <td>-0.103171</td>\n",
              "      <td>0.213918</td>\n",
              "      <td>0.072361</td>\n",
              "      <td>0.257228</td>\n",
              "      <td>-0.328322</td>\n",
              "      <td>0.241558</td>\n",
              "      <td>0.540563</td>\n",
              "      <td>-0.134467</td>\n",
              "      <td>0.100980</td>\n",
              "      <td>0.086089</td>\n",
              "      <td>0.449173</td>\n",
              "      <td>0.029493</td>\n",
              "      <td>-0.314918</td>\n",
              "      <td>0.068761</td>\n",
              "      <td>0.398186</td>\n",
              "      <td>0.301528</td>\n",
              "      <td>0.300199</td>\n",
              "      <td>-0.154410</td>\n",
              "      <td>0.214780</td>\n",
              "      <td>-0.578509</td>\n",
              "      <td>-0.141338</td>\n",
              "      <td>0.304865</td>\n",
              "      <td>-0.125295</td>\n",
              "      <td>0.061498</td>\n",
              "      <td>-0.064937</td>\n",
              "      <td>-0.162696</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.306358</td>\n",
              "      <td>0.198804</td>\n",
              "      <td>-0.473894</td>\n",
              "      <td>0.211371</td>\n",
              "      <td>0.202670</td>\n",
              "      <td>0.210569</td>\n",
              "      <td>0.201893</td>\n",
              "      <td>-0.097871</td>\n",
              "      <td>0.277394</td>\n",
              "      <td>0.270340</td>\n",
              "      <td>0.086187</td>\n",
              "      <td>-0.076706</td>\n",
              "      <td>0.147708</td>\n",
              "      <td>-0.315510</td>\n",
              "      <td>-0.035857</td>\n",
              "      <td>0.532619</td>\n",
              "      <td>0.090852</td>\n",
              "      <td>0.229783</td>\n",
              "      <td>-0.179358</td>\n",
              "      <td>0.431935</td>\n",
              "      <td>-0.354215</td>\n",
              "      <td>-0.124298</td>\n",
              "      <td>0.350391</td>\n",
              "      <td>0.130054</td>\n",
              "      <td>0.074970</td>\n",
              "      <td>-0.180218</td>\n",
              "      <td>-0.075994</td>\n",
              "      <td>0.094390</td>\n",
              "      <td>0.045658</td>\n",
              "      <td>0.534841</td>\n",
              "      <td>0.072381</td>\n",
              "      <td>-0.120653</td>\n",
              "      <td>0.284596</td>\n",
              "      <td>-0.301275</td>\n",
              "      <td>0.169686</td>\n",
              "      <td>0.083162</td>\n",
              "      <td>0.203776</td>\n",
              "      <td>0.352544</td>\n",
              "      <td>0.462114</td>\n",
              "      <td>0.615533</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.374900</td>\n",
              "      <td>0.241235</td>\n",
              "      <td>0.162202</td>\n",
              "      <td>-0.207612</td>\n",
              "      <td>-0.093648</td>\n",
              "      <td>-0.151173</td>\n",
              "      <td>0.119201</td>\n",
              "      <td>-0.152377</td>\n",
              "      <td>0.056225</td>\n",
              "      <td>-0.187376</td>\n",
              "      <td>-0.317524</td>\n",
              "      <td>0.041544</td>\n",
              "      <td>-0.019824</td>\n",
              "      <td>-0.156204</td>\n",
              "      <td>-0.314954</td>\n",
              "      <td>0.229665</td>\n",
              "      <td>-0.000386</td>\n",
              "      <td>0.310775</td>\n",
              "      <td>-0.196551</td>\n",
              "      <td>0.188923</td>\n",
              "      <td>0.582438</td>\n",
              "      <td>-0.096772</td>\n",
              "      <td>0.221287</td>\n",
              "      <td>0.113624</td>\n",
              "      <td>0.414531</td>\n",
              "      <td>-0.050393</td>\n",
              "      <td>-0.370732</td>\n",
              "      <td>0.032574</td>\n",
              "      <td>0.371112</td>\n",
              "      <td>0.296303</td>\n",
              "      <td>0.223358</td>\n",
              "      <td>-0.165383</td>\n",
              "      <td>0.247257</td>\n",
              "      <td>-0.587140</td>\n",
              "      <td>-0.137330</td>\n",
              "      <td>0.212701</td>\n",
              "      <td>-0.126889</td>\n",
              "      <td>0.031823</td>\n",
              "      <td>-0.048240</td>\n",
              "      <td>-0.067000</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.299120</td>\n",
              "      <td>0.205946</td>\n",
              "      <td>-0.497501</td>\n",
              "      <td>0.212194</td>\n",
              "      <td>0.191146</td>\n",
              "      <td>0.209121</td>\n",
              "      <td>0.182551</td>\n",
              "      <td>-0.089321</td>\n",
              "      <td>0.271675</td>\n",
              "      <td>0.274800</td>\n",
              "      <td>0.078810</td>\n",
              "      <td>-0.076359</td>\n",
              "      <td>0.145249</td>\n",
              "      <td>-0.310752</td>\n",
              "      <td>-0.022368</td>\n",
              "      <td>0.529434</td>\n",
              "      <td>0.088469</td>\n",
              "      <td>0.235552</td>\n",
              "      <td>-0.184552</td>\n",
              "      <td>0.441956</td>\n",
              "      <td>-0.364231</td>\n",
              "      <td>-0.127843</td>\n",
              "      <td>0.357312</td>\n",
              "      <td>0.121181</td>\n",
              "      <td>0.071004</td>\n",
              "      <td>-0.178001</td>\n",
              "      <td>-0.066030</td>\n",
              "      <td>0.097680</td>\n",
              "      <td>0.055063</td>\n",
              "      <td>0.546731</td>\n",
              "      <td>0.075838</td>\n",
              "      <td>-0.120019</td>\n",
              "      <td>0.272469</td>\n",
              "      <td>-0.316660</td>\n",
              "      <td>0.168403</td>\n",
              "      <td>0.079594</td>\n",
              "      <td>0.209881</td>\n",
              "      <td>0.351546</td>\n",
              "      <td>0.467715</td>\n",
              "      <td>0.605415</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.384690</td>\n",
              "      <td>0.238071</td>\n",
              "      <td>0.163078</td>\n",
              "      <td>-0.211629</td>\n",
              "      <td>-0.104176</td>\n",
              "      <td>-0.155674</td>\n",
              "      <td>0.116802</td>\n",
              "      <td>-0.159374</td>\n",
              "      <td>0.061421</td>\n",
              "      <td>-0.180406</td>\n",
              "      <td>-0.314748</td>\n",
              "      <td>0.037766</td>\n",
              "      <td>-0.023021</td>\n",
              "      <td>-0.152029</td>\n",
              "      <td>-0.296657</td>\n",
              "      <td>0.223147</td>\n",
              "      <td>0.008299</td>\n",
              "      <td>0.311241</td>\n",
              "      <td>-0.205850</td>\n",
              "      <td>0.192731</td>\n",
              "      <td>0.580908</td>\n",
              "      <td>-0.089537</td>\n",
              "      <td>0.223229</td>\n",
              "      <td>0.116150</td>\n",
              "      <td>0.414811</td>\n",
              "      <td>-0.048666</td>\n",
              "      <td>-0.356069</td>\n",
              "      <td>0.023260</td>\n",
              "      <td>0.379538</td>\n",
              "      <td>0.296504</td>\n",
              "      <td>0.224302</td>\n",
              "      <td>-0.169691</td>\n",
              "      <td>0.246403</td>\n",
              "      <td>-0.602705</td>\n",
              "      <td>-0.138444</td>\n",
              "      <td>0.224174</td>\n",
              "      <td>-0.140794</td>\n",
              "      <td>0.037870</td>\n",
              "      <td>-0.036339</td>\n",
              "      <td>-0.069999</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.300388</td>\n",
              "      <td>0.200988</td>\n",
              "      <td>-0.497074</td>\n",
              "      <td>0.215095</td>\n",
              "      <td>0.198297</td>\n",
              "      <td>0.205976</td>\n",
              "      <td>0.188527</td>\n",
              "      <td>-0.100463</td>\n",
              "      <td>0.260148</td>\n",
              "      <td>0.269592</td>\n",
              "      <td>0.084782</td>\n",
              "      <td>-0.087694</td>\n",
              "      <td>0.159482</td>\n",
              "      <td>-0.303760</td>\n",
              "      <td>-0.028918</td>\n",
              "      <td>0.527010</td>\n",
              "      <td>0.088543</td>\n",
              "      <td>0.233291</td>\n",
              "      <td>-0.180076</td>\n",
              "      <td>0.435514</td>\n",
              "      <td>-0.359726</td>\n",
              "      <td>-0.123712</td>\n",
              "      <td>0.353607</td>\n",
              "      <td>0.117377</td>\n",
              "      <td>0.079083</td>\n",
              "      <td>-0.179347</td>\n",
              "      <td>-0.069081</td>\n",
              "      <td>0.102933</td>\n",
              "      <td>0.040370</td>\n",
              "      <td>0.552164</td>\n",
              "      <td>0.068056</td>\n",
              "      <td>-0.118600</td>\n",
              "      <td>0.277153</td>\n",
              "      <td>-0.312582</td>\n",
              "      <td>0.168025</td>\n",
              "      <td>0.077074</td>\n",
              "      <td>0.211646</td>\n",
              "      <td>0.347595</td>\n",
              "      <td>0.466501</td>\n",
              "      <td>0.613192</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.369939</td>\n",
              "      <td>0.242804</td>\n",
              "      <td>0.165953</td>\n",
              "      <td>-0.200576</td>\n",
              "      <td>-0.091790</td>\n",
              "      <td>-0.149309</td>\n",
              "      <td>0.110276</td>\n",
              "      <td>-0.154901</td>\n",
              "      <td>0.059086</td>\n",
              "      <td>-0.191724</td>\n",
              "      <td>-0.322301</td>\n",
              "      <td>0.039332</td>\n",
              "      <td>-0.014796</td>\n",
              "      <td>-0.142987</td>\n",
              "      <td>-0.314872</td>\n",
              "      <td>0.232114</td>\n",
              "      <td>0.008963</td>\n",
              "      <td>0.314869</td>\n",
              "      <td>-0.200782</td>\n",
              "      <td>0.190683</td>\n",
              "      <td>0.572858</td>\n",
              "      <td>-0.096069</td>\n",
              "      <td>0.227318</td>\n",
              "      <td>0.120489</td>\n",
              "      <td>0.414047</td>\n",
              "      <td>-0.047102</td>\n",
              "      <td>-0.359153</td>\n",
              "      <td>0.024510</td>\n",
              "      <td>0.376216</td>\n",
              "      <td>0.299272</td>\n",
              "      <td>0.216707</td>\n",
              "      <td>-0.164227</td>\n",
              "      <td>0.242545</td>\n",
              "      <td>-0.596216</td>\n",
              "      <td>-0.140515</td>\n",
              "      <td>0.223779</td>\n",
              "      <td>-0.140690</td>\n",
              "      <td>0.040089</td>\n",
              "      <td>-0.035665</td>\n",
              "      <td>-0.064677</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>480</th>\n",
              "      <td>0.296710</td>\n",
              "      <td>0.195684</td>\n",
              "      <td>-0.496675</td>\n",
              "      <td>0.216306</td>\n",
              "      <td>0.198071</td>\n",
              "      <td>0.224041</td>\n",
              "      <td>0.187252</td>\n",
              "      <td>-0.096132</td>\n",
              "      <td>0.254242</td>\n",
              "      <td>0.263849</td>\n",
              "      <td>0.089759</td>\n",
              "      <td>-0.092181</td>\n",
              "      <td>0.172934</td>\n",
              "      <td>-0.303080</td>\n",
              "      <td>-0.032616</td>\n",
              "      <td>0.523828</td>\n",
              "      <td>0.098734</td>\n",
              "      <td>0.223777</td>\n",
              "      <td>-0.187529</td>\n",
              "      <td>0.446112</td>\n",
              "      <td>-0.353280</td>\n",
              "      <td>-0.123781</td>\n",
              "      <td>0.352178</td>\n",
              "      <td>0.109878</td>\n",
              "      <td>0.091999</td>\n",
              "      <td>-0.184220</td>\n",
              "      <td>-0.065154</td>\n",
              "      <td>0.103331</td>\n",
              "      <td>0.033214</td>\n",
              "      <td>0.565335</td>\n",
              "      <td>0.065545</td>\n",
              "      <td>-0.123212</td>\n",
              "      <td>0.261074</td>\n",
              "      <td>-0.312338</td>\n",
              "      <td>0.161678</td>\n",
              "      <td>0.070700</td>\n",
              "      <td>0.218405</td>\n",
              "      <td>0.349611</td>\n",
              "      <td>0.468461</td>\n",
              "      <td>0.607430</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.368061</td>\n",
              "      <td>0.238076</td>\n",
              "      <td>0.167072</td>\n",
              "      <td>-0.195205</td>\n",
              "      <td>-0.080390</td>\n",
              "      <td>-0.143778</td>\n",
              "      <td>0.106464</td>\n",
              "      <td>-0.151259</td>\n",
              "      <td>0.052999</td>\n",
              "      <td>-0.202845</td>\n",
              "      <td>-0.323518</td>\n",
              "      <td>0.037852</td>\n",
              "      <td>-0.007689</td>\n",
              "      <td>-0.145645</td>\n",
              "      <td>-0.304324</td>\n",
              "      <td>0.233517</td>\n",
              "      <td>0.009903</td>\n",
              "      <td>0.317025</td>\n",
              "      <td>-0.193364</td>\n",
              "      <td>0.187509</td>\n",
              "      <td>0.573691</td>\n",
              "      <td>-0.102822</td>\n",
              "      <td>0.229487</td>\n",
              "      <td>0.124587</td>\n",
              "      <td>0.409869</td>\n",
              "      <td>-0.047173</td>\n",
              "      <td>-0.358888</td>\n",
              "      <td>0.024217</td>\n",
              "      <td>0.371099</td>\n",
              "      <td>0.299291</td>\n",
              "      <td>0.221134</td>\n",
              "      <td>-0.169685</td>\n",
              "      <td>0.239862</td>\n",
              "      <td>-0.574564</td>\n",
              "      <td>-0.138788</td>\n",
              "      <td>0.222331</td>\n",
              "      <td>-0.141997</td>\n",
              "      <td>0.036149</td>\n",
              "      <td>-0.032959</td>\n",
              "      <td>-0.056010</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>481</th>\n",
              "      <td>0.301516</td>\n",
              "      <td>0.203763</td>\n",
              "      <td>-0.491835</td>\n",
              "      <td>0.207769</td>\n",
              "      <td>0.196026</td>\n",
              "      <td>0.225036</td>\n",
              "      <td>0.199976</td>\n",
              "      <td>-0.090944</td>\n",
              "      <td>0.267354</td>\n",
              "      <td>0.274369</td>\n",
              "      <td>0.089979</td>\n",
              "      <td>-0.099975</td>\n",
              "      <td>0.170247</td>\n",
              "      <td>-0.314089</td>\n",
              "      <td>-0.042912</td>\n",
              "      <td>0.529263</td>\n",
              "      <td>0.090819</td>\n",
              "      <td>0.223800</td>\n",
              "      <td>-0.186393</td>\n",
              "      <td>0.442406</td>\n",
              "      <td>-0.348135</td>\n",
              "      <td>-0.138589</td>\n",
              "      <td>0.345948</td>\n",
              "      <td>0.104465</td>\n",
              "      <td>0.107658</td>\n",
              "      <td>-0.178120</td>\n",
              "      <td>-0.075733</td>\n",
              "      <td>0.098869</td>\n",
              "      <td>0.034509</td>\n",
              "      <td>0.557425</td>\n",
              "      <td>0.075629</td>\n",
              "      <td>-0.130507</td>\n",
              "      <td>0.273420</td>\n",
              "      <td>-0.301901</td>\n",
              "      <td>0.164346</td>\n",
              "      <td>0.084842</td>\n",
              "      <td>0.217674</td>\n",
              "      <td>0.343790</td>\n",
              "      <td>0.463784</td>\n",
              "      <td>0.607973</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.368820</td>\n",
              "      <td>0.249996</td>\n",
              "      <td>0.168667</td>\n",
              "      <td>-0.197427</td>\n",
              "      <td>-0.080312</td>\n",
              "      <td>-0.136615</td>\n",
              "      <td>0.113891</td>\n",
              "      <td>-0.154556</td>\n",
              "      <td>0.045611</td>\n",
              "      <td>-0.198145</td>\n",
              "      <td>-0.314344</td>\n",
              "      <td>0.033923</td>\n",
              "      <td>-0.007349</td>\n",
              "      <td>-0.151160</td>\n",
              "      <td>-0.298064</td>\n",
              "      <td>0.230776</td>\n",
              "      <td>0.003657</td>\n",
              "      <td>0.323574</td>\n",
              "      <td>-0.185126</td>\n",
              "      <td>0.177546</td>\n",
              "      <td>0.576459</td>\n",
              "      <td>-0.098148</td>\n",
              "      <td>0.237373</td>\n",
              "      <td>0.109329</td>\n",
              "      <td>0.400931</td>\n",
              "      <td>-0.051104</td>\n",
              "      <td>-0.352743</td>\n",
              "      <td>0.036282</td>\n",
              "      <td>0.368991</td>\n",
              "      <td>0.300823</td>\n",
              "      <td>0.227045</td>\n",
              "      <td>-0.170422</td>\n",
              "      <td>0.243435</td>\n",
              "      <td>-0.567498</td>\n",
              "      <td>-0.134042</td>\n",
              "      <td>0.212544</td>\n",
              "      <td>-0.130582</td>\n",
              "      <td>0.045166</td>\n",
              "      <td>-0.029838</td>\n",
              "      <td>-0.055453</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>482</th>\n",
              "      <td>0.302471</td>\n",
              "      <td>0.214367</td>\n",
              "      <td>-0.494262</td>\n",
              "      <td>0.202588</td>\n",
              "      <td>0.199086</td>\n",
              "      <td>0.222635</td>\n",
              "      <td>0.196997</td>\n",
              "      <td>-0.074761</td>\n",
              "      <td>0.278791</td>\n",
              "      <td>0.274174</td>\n",
              "      <td>0.081545</td>\n",
              "      <td>-0.084057</td>\n",
              "      <td>0.154727</td>\n",
              "      <td>-0.324990</td>\n",
              "      <td>-0.043150</td>\n",
              "      <td>0.543074</td>\n",
              "      <td>0.090575</td>\n",
              "      <td>0.226923</td>\n",
              "      <td>-0.179398</td>\n",
              "      <td>0.448532</td>\n",
              "      <td>-0.343554</td>\n",
              "      <td>-0.150017</td>\n",
              "      <td>0.350380</td>\n",
              "      <td>0.112955</td>\n",
              "      <td>0.102471</td>\n",
              "      <td>-0.168925</td>\n",
              "      <td>-0.080275</td>\n",
              "      <td>0.098070</td>\n",
              "      <td>0.054992</td>\n",
              "      <td>0.551028</td>\n",
              "      <td>0.079849</td>\n",
              "      <td>-0.129670</td>\n",
              "      <td>0.276474</td>\n",
              "      <td>-0.301019</td>\n",
              "      <td>0.162053</td>\n",
              "      <td>0.092855</td>\n",
              "      <td>0.212497</td>\n",
              "      <td>0.358718</td>\n",
              "      <td>0.446362</td>\n",
              "      <td>0.606896</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.376246</td>\n",
              "      <td>0.235471</td>\n",
              "      <td>0.158751</td>\n",
              "      <td>-0.210265</td>\n",
              "      <td>-0.094012</td>\n",
              "      <td>-0.149931</td>\n",
              "      <td>0.115405</td>\n",
              "      <td>-0.150614</td>\n",
              "      <td>0.057345</td>\n",
              "      <td>-0.190052</td>\n",
              "      <td>-0.312138</td>\n",
              "      <td>0.032873</td>\n",
              "      <td>-0.012608</td>\n",
              "      <td>-0.154599</td>\n",
              "      <td>-0.292770</td>\n",
              "      <td>0.227776</td>\n",
              "      <td>0.000374</td>\n",
              "      <td>0.306884</td>\n",
              "      <td>-0.203967</td>\n",
              "      <td>0.187159</td>\n",
              "      <td>0.574831</td>\n",
              "      <td>-0.090533</td>\n",
              "      <td>0.228416</td>\n",
              "      <td>0.109242</td>\n",
              "      <td>0.405428</td>\n",
              "      <td>-0.044269</td>\n",
              "      <td>-0.346360</td>\n",
              "      <td>0.034545</td>\n",
              "      <td>0.372968</td>\n",
              "      <td>0.294271</td>\n",
              "      <td>0.239189</td>\n",
              "      <td>-0.169797</td>\n",
              "      <td>0.252185</td>\n",
              "      <td>-0.564015</td>\n",
              "      <td>-0.133615</td>\n",
              "      <td>0.207948</td>\n",
              "      <td>-0.129294</td>\n",
              "      <td>0.043128</td>\n",
              "      <td>-0.037757</td>\n",
              "      <td>-0.063600</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>483</th>\n",
              "      <td>-0.461815</td>\n",
              "      <td>-0.553280</td>\n",
              "      <td>0.244952</td>\n",
              "      <td>0.215400</td>\n",
              "      <td>0.290810</td>\n",
              "      <td>0.535253</td>\n",
              "      <td>0.651677</td>\n",
              "      <td>0.468230</td>\n",
              "      <td>0.654371</td>\n",
              "      <td>-0.067314</td>\n",
              "      <td>-0.220928</td>\n",
              "      <td>0.030706</td>\n",
              "      <td>0.089105</td>\n",
              "      <td>0.705384</td>\n",
              "      <td>0.377641</td>\n",
              "      <td>1.108052</td>\n",
              "      <td>0.566266</td>\n",
              "      <td>0.107145</td>\n",
              "      <td>0.387895</td>\n",
              "      <td>-0.089279</td>\n",
              "      <td>0.354307</td>\n",
              "      <td>0.363212</td>\n",
              "      <td>0.025177</td>\n",
              "      <td>-0.074907</td>\n",
              "      <td>-0.066163</td>\n",
              "      <td>-0.442614</td>\n",
              "      <td>0.319570</td>\n",
              "      <td>-0.029598</td>\n",
              "      <td>0.632027</td>\n",
              "      <td>0.187069</td>\n",
              "      <td>-0.180958</td>\n",
              "      <td>-0.196931</td>\n",
              "      <td>-0.172332</td>\n",
              "      <td>-0.255292</td>\n",
              "      <td>-0.034615</td>\n",
              "      <td>-0.094956</td>\n",
              "      <td>-0.221914</td>\n",
              "      <td>0.734702</td>\n",
              "      <td>-0.097693</td>\n",
              "      <td>0.155587</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.235919</td>\n",
              "      <td>0.560797</td>\n",
              "      <td>0.073279</td>\n",
              "      <td>-0.184731</td>\n",
              "      <td>0.060388</td>\n",
              "      <td>-0.588346</td>\n",
              "      <td>-0.024784</td>\n",
              "      <td>0.300614</td>\n",
              "      <td>0.478370</td>\n",
              "      <td>-0.184600</td>\n",
              "      <td>-0.059539</td>\n",
              "      <td>0.013572</td>\n",
              "      <td>0.447163</td>\n",
              "      <td>-0.273916</td>\n",
              "      <td>0.460695</td>\n",
              "      <td>0.197853</td>\n",
              "      <td>-0.158237</td>\n",
              "      <td>0.383457</td>\n",
              "      <td>0.114069</td>\n",
              "      <td>0.452225</td>\n",
              "      <td>-0.024700</td>\n",
              "      <td>-0.194574</td>\n",
              "      <td>-0.524495</td>\n",
              "      <td>0.286262</td>\n",
              "      <td>0.198166</td>\n",
              "      <td>0.084285</td>\n",
              "      <td>-0.103350</td>\n",
              "      <td>0.174561</td>\n",
              "      <td>0.428523</td>\n",
              "      <td>-0.015782</td>\n",
              "      <td>0.143292</td>\n",
              "      <td>-0.164157</td>\n",
              "      <td>-0.236590</td>\n",
              "      <td>0.541693</td>\n",
              "      <td>0.385318</td>\n",
              "      <td>-0.262914</td>\n",
              "      <td>-0.296878</td>\n",
              "      <td>-0.047433</td>\n",
              "      <td>0.079805</td>\n",
              "      <td>-0.407717</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>484</th>\n",
              "      <td>0.303958</td>\n",
              "      <td>0.204005</td>\n",
              "      <td>-0.492855</td>\n",
              "      <td>0.212926</td>\n",
              "      <td>0.201662</td>\n",
              "      <td>0.219535</td>\n",
              "      <td>0.196432</td>\n",
              "      <td>-0.089491</td>\n",
              "      <td>0.259445</td>\n",
              "      <td>0.267093</td>\n",
              "      <td>0.089099</td>\n",
              "      <td>-0.092213</td>\n",
              "      <td>0.174078</td>\n",
              "      <td>-0.308294</td>\n",
              "      <td>-0.041989</td>\n",
              "      <td>0.528337</td>\n",
              "      <td>0.093601</td>\n",
              "      <td>0.227676</td>\n",
              "      <td>-0.187861</td>\n",
              "      <td>0.440498</td>\n",
              "      <td>-0.352778</td>\n",
              "      <td>-0.126582</td>\n",
              "      <td>0.349474</td>\n",
              "      <td>0.107584</td>\n",
              "      <td>0.096042</td>\n",
              "      <td>-0.177094</td>\n",
              "      <td>-0.072615</td>\n",
              "      <td>0.104924</td>\n",
              "      <td>0.037487</td>\n",
              "      <td>0.560192</td>\n",
              "      <td>0.068068</td>\n",
              "      <td>-0.125337</td>\n",
              "      <td>0.270610</td>\n",
              "      <td>-0.305638</td>\n",
              "      <td>0.161194</td>\n",
              "      <td>0.074836</td>\n",
              "      <td>0.216347</td>\n",
              "      <td>0.352336</td>\n",
              "      <td>0.460768</td>\n",
              "      <td>0.607299</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.371681</td>\n",
              "      <td>0.246770</td>\n",
              "      <td>0.165912</td>\n",
              "      <td>-0.200620</td>\n",
              "      <td>-0.078768</td>\n",
              "      <td>-0.148876</td>\n",
              "      <td>0.107677</td>\n",
              "      <td>-0.151365</td>\n",
              "      <td>0.053284</td>\n",
              "      <td>-0.208637</td>\n",
              "      <td>-0.322215</td>\n",
              "      <td>0.034064</td>\n",
              "      <td>-0.007211</td>\n",
              "      <td>-0.144918</td>\n",
              "      <td>-0.316058</td>\n",
              "      <td>0.239318</td>\n",
              "      <td>0.012685</td>\n",
              "      <td>0.319732</td>\n",
              "      <td>-0.190269</td>\n",
              "      <td>0.194374</td>\n",
              "      <td>0.576231</td>\n",
              "      <td>-0.095763</td>\n",
              "      <td>0.232082</td>\n",
              "      <td>0.113787</td>\n",
              "      <td>0.404261</td>\n",
              "      <td>-0.054956</td>\n",
              "      <td>-0.359050</td>\n",
              "      <td>0.027579</td>\n",
              "      <td>0.364941</td>\n",
              "      <td>0.305099</td>\n",
              "      <td>0.223251</td>\n",
              "      <td>-0.171688</td>\n",
              "      <td>0.245926</td>\n",
              "      <td>-0.571199</td>\n",
              "      <td>-0.139776</td>\n",
              "      <td>0.220691</td>\n",
              "      <td>-0.139388</td>\n",
              "      <td>0.041955</td>\n",
              "      <td>-0.033242</td>\n",
              "      <td>-0.060040</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>485 rows × 768 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "     BERTEmbed_0  BERTEmbed_1  ...  BERTEmbed_766  BERTEmbed_767\n",
              "0       0.301987     0.213715  ...      -0.040448      -0.073326\n",
              "1       0.248635     0.184559  ...      -0.064937      -0.162696\n",
              "2       0.306358     0.198804  ...      -0.048240      -0.067000\n",
              "3       0.299120     0.205946  ...      -0.036339      -0.069999\n",
              "4       0.300388     0.200988  ...      -0.035665      -0.064677\n",
              "..           ...          ...  ...            ...            ...\n",
              "480     0.296710     0.195684  ...      -0.032959      -0.056010\n",
              "481     0.301516     0.203763  ...      -0.029838      -0.055453\n",
              "482     0.302471     0.214367  ...      -0.037757      -0.063600\n",
              "483    -0.461815    -0.553280  ...       0.079805      -0.407717\n",
              "484     0.303958     0.204005  ...      -0.033242      -0.060040\n",
              "\n",
              "[485 rows x 768 columns]"
            ]
          },
          "execution_count": 16,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "lMPu2j75f82A"
      },
      "outputs": [],
      "source": [
        "df_result.to_csv('./data/_PHEMEext_Bert.csv', index = False)\n",
        "# df_result.to_csv('./data/_PHEME_Bert.csv', index = False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Bb0XOyCR9hsk",
        "outputId": "0e33f2ec-48a9-4026-8c58-be9f6f7d843e"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Current number of processed tweets: 0\n",
            "Current number of processed tweets: 300\n",
            "Current number of processed tweets: 600\n",
            "Current number of processed tweets: 900\n",
            "Current number of processed tweets: 1200\n",
            "Current number of processed tweets: 1500\n",
            "Current number of processed tweets: 1800\n",
            "Current number of processed tweets: 2100\n",
            "Current number of processed tweets: 2400\n",
            "Current number of processed tweets: 2700\n",
            "Current number of processed tweets: 3000\n",
            "Current number of processed tweets: 3300\n",
            "Current number of processed tweets: 3600\n",
            "Current number of processed tweets: 3900\n",
            "Current number of processed tweets: 4200\n",
            "Current number of processed tweets: 4500\n",
            "Current number of processed tweets: 4800\n",
            "Current number of processed tweets: 5100\n"
          ]
        }
      ],
      "source": [
        "result = getEmbeddingBERT(bert_classifier,rhi_data)\n",
        "df_result = pd.DataFrame([sent[0].tolist() for sent in result]).add_prefix('BERTEmbed_')"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "fEsaoa9A-IJK"
      },
      "outputs": [],
      "source": [
        "df_result.to_csv('./data/_RHI_Bert.csv', index = False)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 411
        },
        "id": "KVH1rKuAD5i3",
        "outputId": "54f2466c-60e8-433b-a1ca-b1b93e935413"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>BERTEmbed_0</th>\n",
              "      <th>BERTEmbed_1</th>\n",
              "      <th>BERTEmbed_2</th>\n",
              "      <th>BERTEmbed_3</th>\n",
              "      <th>BERTEmbed_4</th>\n",
              "      <th>BERTEmbed_5</th>\n",
              "      <th>BERTEmbed_6</th>\n",
              "      <th>BERTEmbed_7</th>\n",
              "      <th>BERTEmbed_8</th>\n",
              "      <th>BERTEmbed_9</th>\n",
              "      <th>BERTEmbed_10</th>\n",
              "      <th>BERTEmbed_11</th>\n",
              "      <th>BERTEmbed_12</th>\n",
              "      <th>BERTEmbed_13</th>\n",
              "      <th>BERTEmbed_14</th>\n",
              "      <th>BERTEmbed_15</th>\n",
              "      <th>BERTEmbed_16</th>\n",
              "      <th>BERTEmbed_17</th>\n",
              "      <th>BERTEmbed_18</th>\n",
              "      <th>BERTEmbed_19</th>\n",
              "      <th>BERTEmbed_20</th>\n",
              "      <th>BERTEmbed_21</th>\n",
              "      <th>BERTEmbed_22</th>\n",
              "      <th>BERTEmbed_23</th>\n",
              "      <th>BERTEmbed_24</th>\n",
              "      <th>BERTEmbed_25</th>\n",
              "      <th>BERTEmbed_26</th>\n",
              "      <th>BERTEmbed_27</th>\n",
              "      <th>BERTEmbed_28</th>\n",
              "      <th>BERTEmbed_29</th>\n",
              "      <th>BERTEmbed_30</th>\n",
              "      <th>BERTEmbed_31</th>\n",
              "      <th>BERTEmbed_32</th>\n",
              "      <th>BERTEmbed_33</th>\n",
              "      <th>BERTEmbed_34</th>\n",
              "      <th>BERTEmbed_35</th>\n",
              "      <th>BERTEmbed_36</th>\n",
              "      <th>BERTEmbed_37</th>\n",
              "      <th>BERTEmbed_38</th>\n",
              "      <th>BERTEmbed_39</th>\n",
              "      <th>...</th>\n",
              "      <th>BERTEmbed_728</th>\n",
              "      <th>BERTEmbed_729</th>\n",
              "      <th>BERTEmbed_730</th>\n",
              "      <th>BERTEmbed_731</th>\n",
              "      <th>BERTEmbed_732</th>\n",
              "      <th>BERTEmbed_733</th>\n",
              "      <th>BERTEmbed_734</th>\n",
              "      <th>BERTEmbed_735</th>\n",
              "      <th>BERTEmbed_736</th>\n",
              "      <th>BERTEmbed_737</th>\n",
              "      <th>BERTEmbed_738</th>\n",
              "      <th>BERTEmbed_739</th>\n",
              "      <th>BERTEmbed_740</th>\n",
              "      <th>BERTEmbed_741</th>\n",
              "      <th>BERTEmbed_742</th>\n",
              "      <th>BERTEmbed_743</th>\n",
              "      <th>BERTEmbed_744</th>\n",
              "      <th>BERTEmbed_745</th>\n",
              "      <th>BERTEmbed_746</th>\n",
              "      <th>BERTEmbed_747</th>\n",
              "      <th>BERTEmbed_748</th>\n",
              "      <th>BERTEmbed_749</th>\n",
              "      <th>BERTEmbed_750</th>\n",
              "      <th>BERTEmbed_751</th>\n",
              "      <th>BERTEmbed_752</th>\n",
              "      <th>BERTEmbed_753</th>\n",
              "      <th>BERTEmbed_754</th>\n",
              "      <th>BERTEmbed_755</th>\n",
              "      <th>BERTEmbed_756</th>\n",
              "      <th>BERTEmbed_757</th>\n",
              "      <th>BERTEmbed_758</th>\n",
              "      <th>BERTEmbed_759</th>\n",
              "      <th>BERTEmbed_760</th>\n",
              "      <th>BERTEmbed_761</th>\n",
              "      <th>BERTEmbed_762</th>\n",
              "      <th>BERTEmbed_763</th>\n",
              "      <th>BERTEmbed_764</th>\n",
              "      <th>BERTEmbed_765</th>\n",
              "      <th>BERTEmbed_766</th>\n",
              "      <th>BERTEmbed_767</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0.297946</td>\n",
              "      <td>0.203994</td>\n",
              "      <td>-0.499056</td>\n",
              "      <td>0.213547</td>\n",
              "      <td>0.196660</td>\n",
              "      <td>0.227667</td>\n",
              "      <td>0.194849</td>\n",
              "      <td>-0.093375</td>\n",
              "      <td>0.258162</td>\n",
              "      <td>0.271142</td>\n",
              "      <td>0.090113</td>\n",
              "      <td>-0.092884</td>\n",
              "      <td>0.176856</td>\n",
              "      <td>-0.307920</td>\n",
              "      <td>-0.038434</td>\n",
              "      <td>0.519788</td>\n",
              "      <td>0.095166</td>\n",
              "      <td>0.223867</td>\n",
              "      <td>-0.182025</td>\n",
              "      <td>0.444022</td>\n",
              "      <td>-0.352069</td>\n",
              "      <td>-0.131495</td>\n",
              "      <td>0.351419</td>\n",
              "      <td>0.111201</td>\n",
              "      <td>0.084987</td>\n",
              "      <td>-0.185981</td>\n",
              "      <td>-0.066530</td>\n",
              "      <td>0.101560</td>\n",
              "      <td>0.034992</td>\n",
              "      <td>0.559159</td>\n",
              "      <td>0.065297</td>\n",
              "      <td>-0.123398</td>\n",
              "      <td>0.266490</td>\n",
              "      <td>-0.313069</td>\n",
              "      <td>0.171754</td>\n",
              "      <td>0.072665</td>\n",
              "      <td>0.217715</td>\n",
              "      <td>0.337471</td>\n",
              "      <td>0.472570</td>\n",
              "      <td>0.611313</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.376209</td>\n",
              "      <td>0.246628</td>\n",
              "      <td>0.161467</td>\n",
              "      <td>-0.198552</td>\n",
              "      <td>-0.088427</td>\n",
              "      <td>-0.137050</td>\n",
              "      <td>0.110910</td>\n",
              "      <td>-0.162978</td>\n",
              "      <td>0.057977</td>\n",
              "      <td>-0.195143</td>\n",
              "      <td>-0.317914</td>\n",
              "      <td>0.041483</td>\n",
              "      <td>-0.013151</td>\n",
              "      <td>-0.142792</td>\n",
              "      <td>-0.302807</td>\n",
              "      <td>0.239212</td>\n",
              "      <td>0.007527</td>\n",
              "      <td>0.319393</td>\n",
              "      <td>-0.195951</td>\n",
              "      <td>0.181913</td>\n",
              "      <td>0.575082</td>\n",
              "      <td>-0.099842</td>\n",
              "      <td>0.236463</td>\n",
              "      <td>0.124098</td>\n",
              "      <td>0.404514</td>\n",
              "      <td>-0.050513</td>\n",
              "      <td>-0.355765</td>\n",
              "      <td>0.024739</td>\n",
              "      <td>0.370513</td>\n",
              "      <td>0.301253</td>\n",
              "      <td>0.214088</td>\n",
              "      <td>-0.166805</td>\n",
              "      <td>0.239269</td>\n",
              "      <td>-0.577732</td>\n",
              "      <td>-0.140498</td>\n",
              "      <td>0.219537</td>\n",
              "      <td>-0.138748</td>\n",
              "      <td>0.044908</td>\n",
              "      <td>-0.035265</td>\n",
              "      <td>-0.057258</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>0.295606</td>\n",
              "      <td>0.204244</td>\n",
              "      <td>-0.499383</td>\n",
              "      <td>0.210717</td>\n",
              "      <td>0.198588</td>\n",
              "      <td>0.227903</td>\n",
              "      <td>0.189495</td>\n",
              "      <td>-0.085283</td>\n",
              "      <td>0.260716</td>\n",
              "      <td>0.269651</td>\n",
              "      <td>0.099998</td>\n",
              "      <td>-0.100179</td>\n",
              "      <td>0.158110</td>\n",
              "      <td>-0.319833</td>\n",
              "      <td>-0.041735</td>\n",
              "      <td>0.529102</td>\n",
              "      <td>0.092172</td>\n",
              "      <td>0.234434</td>\n",
              "      <td>-0.189306</td>\n",
              "      <td>0.442956</td>\n",
              "      <td>-0.355478</td>\n",
              "      <td>-0.141370</td>\n",
              "      <td>0.357359</td>\n",
              "      <td>0.109652</td>\n",
              "      <td>0.099337</td>\n",
              "      <td>-0.174226</td>\n",
              "      <td>-0.081724</td>\n",
              "      <td>0.108346</td>\n",
              "      <td>0.048244</td>\n",
              "      <td>0.554192</td>\n",
              "      <td>0.075846</td>\n",
              "      <td>-0.121724</td>\n",
              "      <td>0.275677</td>\n",
              "      <td>-0.297837</td>\n",
              "      <td>0.161615</td>\n",
              "      <td>0.075789</td>\n",
              "      <td>0.220245</td>\n",
              "      <td>0.349307</td>\n",
              "      <td>0.450045</td>\n",
              "      <td>0.606146</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.361443</td>\n",
              "      <td>0.241416</td>\n",
              "      <td>0.171471</td>\n",
              "      <td>-0.201332</td>\n",
              "      <td>-0.086248</td>\n",
              "      <td>-0.152632</td>\n",
              "      <td>0.113391</td>\n",
              "      <td>-0.154615</td>\n",
              "      <td>0.062096</td>\n",
              "      <td>-0.199849</td>\n",
              "      <td>-0.325092</td>\n",
              "      <td>0.029384</td>\n",
              "      <td>-0.009026</td>\n",
              "      <td>-0.164724</td>\n",
              "      <td>-0.304481</td>\n",
              "      <td>0.232149</td>\n",
              "      <td>0.004177</td>\n",
              "      <td>0.304888</td>\n",
              "      <td>-0.201973</td>\n",
              "      <td>0.197971</td>\n",
              "      <td>0.567020</td>\n",
              "      <td>-0.106541</td>\n",
              "      <td>0.238295</td>\n",
              "      <td>0.126391</td>\n",
              "      <td>0.411974</td>\n",
              "      <td>-0.037872</td>\n",
              "      <td>-0.355570</td>\n",
              "      <td>0.039528</td>\n",
              "      <td>0.372111</td>\n",
              "      <td>0.300876</td>\n",
              "      <td>0.215221</td>\n",
              "      <td>-0.160260</td>\n",
              "      <td>0.242425</td>\n",
              "      <td>-0.566059</td>\n",
              "      <td>-0.133051</td>\n",
              "      <td>0.218862</td>\n",
              "      <td>-0.135239</td>\n",
              "      <td>0.041871</td>\n",
              "      <td>-0.042629</td>\n",
              "      <td>-0.063909</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0.301102</td>\n",
              "      <td>0.203472</td>\n",
              "      <td>-0.492820</td>\n",
              "      <td>0.203284</td>\n",
              "      <td>0.199285</td>\n",
              "      <td>0.212873</td>\n",
              "      <td>0.188556</td>\n",
              "      <td>-0.086498</td>\n",
              "      <td>0.285209</td>\n",
              "      <td>0.269128</td>\n",
              "      <td>0.091030</td>\n",
              "      <td>-0.080946</td>\n",
              "      <td>0.136123</td>\n",
              "      <td>-0.321619</td>\n",
              "      <td>-0.040465</td>\n",
              "      <td>0.542610</td>\n",
              "      <td>0.098858</td>\n",
              "      <td>0.240241</td>\n",
              "      <td>-0.183112</td>\n",
              "      <td>0.436210</td>\n",
              "      <td>-0.352844</td>\n",
              "      <td>-0.138281</td>\n",
              "      <td>0.355719</td>\n",
              "      <td>0.133165</td>\n",
              "      <td>0.087124</td>\n",
              "      <td>-0.155789</td>\n",
              "      <td>-0.086958</td>\n",
              "      <td>0.097145</td>\n",
              "      <td>0.061762</td>\n",
              "      <td>0.537655</td>\n",
              "      <td>0.079842</td>\n",
              "      <td>-0.120110</td>\n",
              "      <td>0.281972</td>\n",
              "      <td>-0.303754</td>\n",
              "      <td>0.162821</td>\n",
              "      <td>0.090998</td>\n",
              "      <td>0.208587</td>\n",
              "      <td>0.357905</td>\n",
              "      <td>0.437511</td>\n",
              "      <td>0.611550</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.367431</td>\n",
              "      <td>0.227422</td>\n",
              "      <td>0.170631</td>\n",
              "      <td>-0.208352</td>\n",
              "      <td>-0.099096</td>\n",
              "      <td>-0.161995</td>\n",
              "      <td>0.121463</td>\n",
              "      <td>-0.153550</td>\n",
              "      <td>0.064215</td>\n",
              "      <td>-0.179611</td>\n",
              "      <td>-0.308386</td>\n",
              "      <td>0.031840</td>\n",
              "      <td>-0.017500</td>\n",
              "      <td>-0.183257</td>\n",
              "      <td>-0.292693</td>\n",
              "      <td>0.225274</td>\n",
              "      <td>-0.014289</td>\n",
              "      <td>0.293642</td>\n",
              "      <td>-0.205405</td>\n",
              "      <td>0.190455</td>\n",
              "      <td>0.565723</td>\n",
              "      <td>-0.085674</td>\n",
              "      <td>0.217515</td>\n",
              "      <td>0.116086</td>\n",
              "      <td>0.417646</td>\n",
              "      <td>-0.028992</td>\n",
              "      <td>-0.349879</td>\n",
              "      <td>0.048025</td>\n",
              "      <td>0.374956</td>\n",
              "      <td>0.291391</td>\n",
              "      <td>0.224222</td>\n",
              "      <td>-0.163860</td>\n",
              "      <td>0.252944</td>\n",
              "      <td>-0.577759</td>\n",
              "      <td>-0.128209</td>\n",
              "      <td>0.210840</td>\n",
              "      <td>-0.126757</td>\n",
              "      <td>0.029583</td>\n",
              "      <td>-0.050423</td>\n",
              "      <td>-0.071233</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>0.298717</td>\n",
              "      <td>0.208594</td>\n",
              "      <td>-0.495822</td>\n",
              "      <td>0.208106</td>\n",
              "      <td>0.197800</td>\n",
              "      <td>0.225167</td>\n",
              "      <td>0.201372</td>\n",
              "      <td>-0.093187</td>\n",
              "      <td>0.261350</td>\n",
              "      <td>0.268694</td>\n",
              "      <td>0.088287</td>\n",
              "      <td>-0.094689</td>\n",
              "      <td>0.165091</td>\n",
              "      <td>-0.319498</td>\n",
              "      <td>-0.042514</td>\n",
              "      <td>0.535865</td>\n",
              "      <td>0.096159</td>\n",
              "      <td>0.224006</td>\n",
              "      <td>-0.173223</td>\n",
              "      <td>0.439430</td>\n",
              "      <td>-0.350504</td>\n",
              "      <td>-0.144939</td>\n",
              "      <td>0.350333</td>\n",
              "      <td>0.106904</td>\n",
              "      <td>0.109094</td>\n",
              "      <td>-0.167813</td>\n",
              "      <td>-0.084326</td>\n",
              "      <td>0.103470</td>\n",
              "      <td>0.035739</td>\n",
              "      <td>0.557059</td>\n",
              "      <td>0.077050</td>\n",
              "      <td>-0.127826</td>\n",
              "      <td>0.278076</td>\n",
              "      <td>-0.301439</td>\n",
              "      <td>0.161258</td>\n",
              "      <td>0.088135</td>\n",
              "      <td>0.218025</td>\n",
              "      <td>0.352538</td>\n",
              "      <td>0.448241</td>\n",
              "      <td>0.604920</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.366874</td>\n",
              "      <td>0.250117</td>\n",
              "      <td>0.167595</td>\n",
              "      <td>-0.205010</td>\n",
              "      <td>-0.085647</td>\n",
              "      <td>-0.143952</td>\n",
              "      <td>0.109110</td>\n",
              "      <td>-0.146660</td>\n",
              "      <td>0.057317</td>\n",
              "      <td>-0.196542</td>\n",
              "      <td>-0.317860</td>\n",
              "      <td>0.024818</td>\n",
              "      <td>-0.002563</td>\n",
              "      <td>-0.146019</td>\n",
              "      <td>-0.306179</td>\n",
              "      <td>0.231716</td>\n",
              "      <td>0.006044</td>\n",
              "      <td>0.314065</td>\n",
              "      <td>-0.203201</td>\n",
              "      <td>0.182983</td>\n",
              "      <td>0.567688</td>\n",
              "      <td>-0.099277</td>\n",
              "      <td>0.236405</td>\n",
              "      <td>0.115684</td>\n",
              "      <td>0.408776</td>\n",
              "      <td>-0.036635</td>\n",
              "      <td>-0.350819</td>\n",
              "      <td>0.040170</td>\n",
              "      <td>0.370420</td>\n",
              "      <td>0.296418</td>\n",
              "      <td>0.225640</td>\n",
              "      <td>-0.166886</td>\n",
              "      <td>0.242574</td>\n",
              "      <td>-0.569904</td>\n",
              "      <td>-0.132144</td>\n",
              "      <td>0.216476</td>\n",
              "      <td>-0.135107</td>\n",
              "      <td>0.043263</td>\n",
              "      <td>-0.031238</td>\n",
              "      <td>-0.058959</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>0.301026</td>\n",
              "      <td>0.203712</td>\n",
              "      <td>-0.497528</td>\n",
              "      <td>0.211088</td>\n",
              "      <td>0.192684</td>\n",
              "      <td>0.225895</td>\n",
              "      <td>0.191089</td>\n",
              "      <td>-0.091480</td>\n",
              "      <td>0.268301</td>\n",
              "      <td>0.265148</td>\n",
              "      <td>0.090018</td>\n",
              "      <td>-0.091663</td>\n",
              "      <td>0.163100</td>\n",
              "      <td>-0.315002</td>\n",
              "      <td>-0.042382</td>\n",
              "      <td>0.526398</td>\n",
              "      <td>0.098250</td>\n",
              "      <td>0.225047</td>\n",
              "      <td>-0.183419</td>\n",
              "      <td>0.442693</td>\n",
              "      <td>-0.351429</td>\n",
              "      <td>-0.135175</td>\n",
              "      <td>0.351086</td>\n",
              "      <td>0.114343</td>\n",
              "      <td>0.090603</td>\n",
              "      <td>-0.173736</td>\n",
              "      <td>-0.072745</td>\n",
              "      <td>0.102957</td>\n",
              "      <td>0.044489</td>\n",
              "      <td>0.555536</td>\n",
              "      <td>0.073358</td>\n",
              "      <td>-0.127452</td>\n",
              "      <td>0.271576</td>\n",
              "      <td>-0.304996</td>\n",
              "      <td>0.164616</td>\n",
              "      <td>0.078614</td>\n",
              "      <td>0.215787</td>\n",
              "      <td>0.348520</td>\n",
              "      <td>0.459276</td>\n",
              "      <td>0.608789</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.370301</td>\n",
              "      <td>0.237411</td>\n",
              "      <td>0.164621</td>\n",
              "      <td>-0.201419</td>\n",
              "      <td>-0.090248</td>\n",
              "      <td>-0.145942</td>\n",
              "      <td>0.113642</td>\n",
              "      <td>-0.154056</td>\n",
              "      <td>0.055786</td>\n",
              "      <td>-0.191710</td>\n",
              "      <td>-0.316176</td>\n",
              "      <td>0.034922</td>\n",
              "      <td>-0.011003</td>\n",
              "      <td>-0.152098</td>\n",
              "      <td>-0.297838</td>\n",
              "      <td>0.230448</td>\n",
              "      <td>-0.000034</td>\n",
              "      <td>0.308018</td>\n",
              "      <td>-0.194742</td>\n",
              "      <td>0.185629</td>\n",
              "      <td>0.572134</td>\n",
              "      <td>-0.096859</td>\n",
              "      <td>0.229449</td>\n",
              "      <td>0.118000</td>\n",
              "      <td>0.409043</td>\n",
              "      <td>-0.041848</td>\n",
              "      <td>-0.352134</td>\n",
              "      <td>0.034900</td>\n",
              "      <td>0.372045</td>\n",
              "      <td>0.297666</td>\n",
              "      <td>0.219349</td>\n",
              "      <td>-0.166691</td>\n",
              "      <td>0.243281</td>\n",
              "      <td>-0.574679</td>\n",
              "      <td>-0.136919</td>\n",
              "      <td>0.218606</td>\n",
              "      <td>-0.133076</td>\n",
              "      <td>0.038746</td>\n",
              "      <td>-0.040579</td>\n",
              "      <td>-0.059467</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>...</th>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "      <td>...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5222</th>\n",
              "      <td>-0.161756</td>\n",
              "      <td>-0.448114</td>\n",
              "      <td>-0.194898</td>\n",
              "      <td>0.166567</td>\n",
              "      <td>0.523893</td>\n",
              "      <td>0.298642</td>\n",
              "      <td>-0.004362</td>\n",
              "      <td>0.376439</td>\n",
              "      <td>0.777519</td>\n",
              "      <td>-0.063401</td>\n",
              "      <td>0.182189</td>\n",
              "      <td>-0.172003</td>\n",
              "      <td>-0.122013</td>\n",
              "      <td>0.222334</td>\n",
              "      <td>0.316154</td>\n",
              "      <td>0.919189</td>\n",
              "      <td>0.760110</td>\n",
              "      <td>0.455591</td>\n",
              "      <td>-0.003305</td>\n",
              "      <td>0.284072</td>\n",
              "      <td>0.049668</td>\n",
              "      <td>-0.023914</td>\n",
              "      <td>0.492080</td>\n",
              "      <td>-0.105371</td>\n",
              "      <td>0.056435</td>\n",
              "      <td>-0.132467</td>\n",
              "      <td>-0.010618</td>\n",
              "      <td>0.111524</td>\n",
              "      <td>0.640192</td>\n",
              "      <td>0.280732</td>\n",
              "      <td>-0.042401</td>\n",
              "      <td>-0.191462</td>\n",
              "      <td>0.089892</td>\n",
              "      <td>-0.082324</td>\n",
              "      <td>-0.051619</td>\n",
              "      <td>0.234865</td>\n",
              "      <td>-0.123215</td>\n",
              "      <td>0.692746</td>\n",
              "      <td>-0.105865</td>\n",
              "      <td>0.491340</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.261227</td>\n",
              "      <td>0.360298</td>\n",
              "      <td>0.295889</td>\n",
              "      <td>-0.250969</td>\n",
              "      <td>0.076526</td>\n",
              "      <td>-0.381980</td>\n",
              "      <td>-0.066263</td>\n",
              "      <td>0.219656</td>\n",
              "      <td>0.382665</td>\n",
              "      <td>-0.046832</td>\n",
              "      <td>-0.139894</td>\n",
              "      <td>0.040498</td>\n",
              "      <td>0.514541</td>\n",
              "      <td>-0.121717</td>\n",
              "      <td>0.443030</td>\n",
              "      <td>0.243352</td>\n",
              "      <td>-0.137723</td>\n",
              "      <td>0.061313</td>\n",
              "      <td>-0.337111</td>\n",
              "      <td>0.532772</td>\n",
              "      <td>0.036736</td>\n",
              "      <td>0.022951</td>\n",
              "      <td>-0.284968</td>\n",
              "      <td>0.122532</td>\n",
              "      <td>0.808282</td>\n",
              "      <td>-0.036924</td>\n",
              "      <td>-0.164414</td>\n",
              "      <td>0.232579</td>\n",
              "      <td>0.642230</td>\n",
              "      <td>0.314379</td>\n",
              "      <td>0.185931</td>\n",
              "      <td>-0.194871</td>\n",
              "      <td>-0.090293</td>\n",
              "      <td>0.157510</td>\n",
              "      <td>0.082789</td>\n",
              "      <td>0.067447</td>\n",
              "      <td>-0.157769</td>\n",
              "      <td>0.065260</td>\n",
              "      <td>0.022575</td>\n",
              "      <td>-0.495942</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5223</th>\n",
              "      <td>-0.593530</td>\n",
              "      <td>-0.523108</td>\n",
              "      <td>0.104469</td>\n",
              "      <td>0.305812</td>\n",
              "      <td>0.335130</td>\n",
              "      <td>0.430348</td>\n",
              "      <td>0.567081</td>\n",
              "      <td>0.476604</td>\n",
              "      <td>0.797814</td>\n",
              "      <td>-0.104125</td>\n",
              "      <td>0.003070</td>\n",
              "      <td>-0.091338</td>\n",
              "      <td>0.011470</td>\n",
              "      <td>0.498766</td>\n",
              "      <td>0.208280</td>\n",
              "      <td>1.276119</td>\n",
              "      <td>0.691798</td>\n",
              "      <td>0.141713</td>\n",
              "      <td>0.361216</td>\n",
              "      <td>0.036403</td>\n",
              "      <td>0.168782</td>\n",
              "      <td>0.174348</td>\n",
              "      <td>-0.038981</td>\n",
              "      <td>-0.091984</td>\n",
              "      <td>-0.046420</td>\n",
              "      <td>-0.433114</td>\n",
              "      <td>0.232216</td>\n",
              "      <td>0.017375</td>\n",
              "      <td>0.373732</td>\n",
              "      <td>0.208637</td>\n",
              "      <td>-0.084850</td>\n",
              "      <td>-0.129077</td>\n",
              "      <td>-0.119808</td>\n",
              "      <td>-0.365497</td>\n",
              "      <td>-0.045794</td>\n",
              "      <td>0.017812</td>\n",
              "      <td>-0.148036</td>\n",
              "      <td>0.784663</td>\n",
              "      <td>-0.104199</td>\n",
              "      <td>0.253858</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.294621</td>\n",
              "      <td>0.648357</td>\n",
              "      <td>0.181563</td>\n",
              "      <td>-0.312259</td>\n",
              "      <td>0.064640</td>\n",
              "      <td>-0.573107</td>\n",
              "      <td>-0.010907</td>\n",
              "      <td>0.342248</td>\n",
              "      <td>0.425883</td>\n",
              "      <td>-0.216878</td>\n",
              "      <td>-0.038760</td>\n",
              "      <td>-0.057925</td>\n",
              "      <td>0.519517</td>\n",
              "      <td>-0.287014</td>\n",
              "      <td>0.536025</td>\n",
              "      <td>0.314241</td>\n",
              "      <td>-0.087154</td>\n",
              "      <td>0.303494</td>\n",
              "      <td>-0.084919</td>\n",
              "      <td>0.638656</td>\n",
              "      <td>-0.095671</td>\n",
              "      <td>-0.236405</td>\n",
              "      <td>-0.532938</td>\n",
              "      <td>0.443283</td>\n",
              "      <td>0.435793</td>\n",
              "      <td>0.071228</td>\n",
              "      <td>-0.087837</td>\n",
              "      <td>0.051851</td>\n",
              "      <td>0.383170</td>\n",
              "      <td>0.141366</td>\n",
              "      <td>0.145616</td>\n",
              "      <td>-0.200405</td>\n",
              "      <td>-0.166344</td>\n",
              "      <td>0.591608</td>\n",
              "      <td>0.356316</td>\n",
              "      <td>-0.231047</td>\n",
              "      <td>-0.179833</td>\n",
              "      <td>-0.062807</td>\n",
              "      <td>-0.006910</td>\n",
              "      <td>-0.514361</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5224</th>\n",
              "      <td>0.300492</td>\n",
              "      <td>0.191589</td>\n",
              "      <td>-0.500008</td>\n",
              "      <td>0.216377</td>\n",
              "      <td>0.199878</td>\n",
              "      <td>0.226429</td>\n",
              "      <td>0.188519</td>\n",
              "      <td>-0.102770</td>\n",
              "      <td>0.254270</td>\n",
              "      <td>0.261093</td>\n",
              "      <td>0.094089</td>\n",
              "      <td>-0.092076</td>\n",
              "      <td>0.188594</td>\n",
              "      <td>-0.292861</td>\n",
              "      <td>-0.027486</td>\n",
              "      <td>0.528188</td>\n",
              "      <td>0.101867</td>\n",
              "      <td>0.220473</td>\n",
              "      <td>-0.185013</td>\n",
              "      <td>0.445310</td>\n",
              "      <td>-0.358039</td>\n",
              "      <td>-0.119082</td>\n",
              "      <td>0.353812</td>\n",
              "      <td>0.109652</td>\n",
              "      <td>0.092463</td>\n",
              "      <td>-0.185271</td>\n",
              "      <td>-0.059294</td>\n",
              "      <td>0.105419</td>\n",
              "      <td>0.035044</td>\n",
              "      <td>0.572742</td>\n",
              "      <td>0.058562</td>\n",
              "      <td>-0.114848</td>\n",
              "      <td>0.263776</td>\n",
              "      <td>-0.321319</td>\n",
              "      <td>0.161118</td>\n",
              "      <td>0.071945</td>\n",
              "      <td>0.214801</td>\n",
              "      <td>0.350749</td>\n",
              "      <td>0.473799</td>\n",
              "      <td>0.612233</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.368823</td>\n",
              "      <td>0.233690</td>\n",
              "      <td>0.168820</td>\n",
              "      <td>-0.189202</td>\n",
              "      <td>-0.077514</td>\n",
              "      <td>-0.136362</td>\n",
              "      <td>0.101767</td>\n",
              "      <td>-0.145810</td>\n",
              "      <td>0.050650</td>\n",
              "      <td>-0.202628</td>\n",
              "      <td>-0.322789</td>\n",
              "      <td>0.049984</td>\n",
              "      <td>-0.004759</td>\n",
              "      <td>-0.142675</td>\n",
              "      <td>-0.305112</td>\n",
              "      <td>0.236501</td>\n",
              "      <td>0.018629</td>\n",
              "      <td>0.316934</td>\n",
              "      <td>-0.188784</td>\n",
              "      <td>0.193418</td>\n",
              "      <td>0.567894</td>\n",
              "      <td>-0.101717</td>\n",
              "      <td>0.233139</td>\n",
              "      <td>0.129274</td>\n",
              "      <td>0.419129</td>\n",
              "      <td>-0.043680</td>\n",
              "      <td>-0.357719</td>\n",
              "      <td>0.020728</td>\n",
              "      <td>0.373722</td>\n",
              "      <td>0.295953</td>\n",
              "      <td>0.216574</td>\n",
              "      <td>-0.165701</td>\n",
              "      <td>0.242660</td>\n",
              "      <td>-0.572967</td>\n",
              "      <td>-0.137008</td>\n",
              "      <td>0.220005</td>\n",
              "      <td>-0.139921</td>\n",
              "      <td>0.043865</td>\n",
              "      <td>-0.031499</td>\n",
              "      <td>-0.057095</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5225</th>\n",
              "      <td>-0.004848</td>\n",
              "      <td>-0.034025</td>\n",
              "      <td>-0.355564</td>\n",
              "      <td>0.216482</td>\n",
              "      <td>0.352537</td>\n",
              "      <td>0.224281</td>\n",
              "      <td>0.035540</td>\n",
              "      <td>0.110744</td>\n",
              "      <td>0.582603</td>\n",
              "      <td>0.224755</td>\n",
              "      <td>0.264491</td>\n",
              "      <td>-0.133408</td>\n",
              "      <td>-0.207846</td>\n",
              "      <td>-0.346848</td>\n",
              "      <td>0.112305</td>\n",
              "      <td>0.634083</td>\n",
              "      <td>0.256311</td>\n",
              "      <td>0.350663</td>\n",
              "      <td>-0.001356</td>\n",
              "      <td>0.435956</td>\n",
              "      <td>-0.387553</td>\n",
              "      <td>-0.332673</td>\n",
              "      <td>0.441690</td>\n",
              "      <td>0.011856</td>\n",
              "      <td>0.171681</td>\n",
              "      <td>-0.112015</td>\n",
              "      <td>-0.199945</td>\n",
              "      <td>0.037816</td>\n",
              "      <td>0.128880</td>\n",
              "      <td>0.185116</td>\n",
              "      <td>0.191942</td>\n",
              "      <td>-0.047886</td>\n",
              "      <td>0.176530</td>\n",
              "      <td>-0.324812</td>\n",
              "      <td>0.165196</td>\n",
              "      <td>0.114447</td>\n",
              "      <td>0.103238</td>\n",
              "      <td>0.333215</td>\n",
              "      <td>0.102361</td>\n",
              "      <td>0.617350</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.278933</td>\n",
              "      <td>0.044017</td>\n",
              "      <td>0.323226</td>\n",
              "      <td>-0.258471</td>\n",
              "      <td>-0.076016</td>\n",
              "      <td>-0.287672</td>\n",
              "      <td>0.098841</td>\n",
              "      <td>-0.042089</td>\n",
              "      <td>0.261028</td>\n",
              "      <td>-0.127222</td>\n",
              "      <td>-0.186732</td>\n",
              "      <td>-0.102074</td>\n",
              "      <td>0.261124</td>\n",
              "      <td>-0.263056</td>\n",
              "      <td>0.142129</td>\n",
              "      <td>0.115485</td>\n",
              "      <td>-0.034462</td>\n",
              "      <td>0.069779</td>\n",
              "      <td>-0.492992</td>\n",
              "      <td>0.275766</td>\n",
              "      <td>0.281265</td>\n",
              "      <td>-0.152669</td>\n",
              "      <td>0.123695</td>\n",
              "      <td>0.264068</td>\n",
              "      <td>0.538221</td>\n",
              "      <td>0.214268</td>\n",
              "      <td>-0.257985</td>\n",
              "      <td>0.192074</td>\n",
              "      <td>0.479494</td>\n",
              "      <td>0.242291</td>\n",
              "      <td>0.344083</td>\n",
              "      <td>-0.144548</td>\n",
              "      <td>0.155721</td>\n",
              "      <td>-0.216285</td>\n",
              "      <td>0.001447</td>\n",
              "      <td>0.188301</td>\n",
              "      <td>0.021298</td>\n",
              "      <td>0.013169</td>\n",
              "      <td>-0.148900</td>\n",
              "      <td>-0.236210</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>5226</th>\n",
              "      <td>0.303914</td>\n",
              "      <td>0.201205</td>\n",
              "      <td>-0.494514</td>\n",
              "      <td>0.213323</td>\n",
              "      <td>0.198750</td>\n",
              "      <td>0.219460</td>\n",
              "      <td>0.191517</td>\n",
              "      <td>-0.097005</td>\n",
              "      <td>0.275173</td>\n",
              "      <td>0.265436</td>\n",
              "      <td>0.090410</td>\n",
              "      <td>-0.082908</td>\n",
              "      <td>0.168315</td>\n",
              "      <td>-0.305000</td>\n",
              "      <td>-0.036986</td>\n",
              "      <td>0.528776</td>\n",
              "      <td>0.097420</td>\n",
              "      <td>0.227068</td>\n",
              "      <td>-0.177594</td>\n",
              "      <td>0.441327</td>\n",
              "      <td>-0.353551</td>\n",
              "      <td>-0.125400</td>\n",
              "      <td>0.356122</td>\n",
              "      <td>0.120030</td>\n",
              "      <td>0.081991</td>\n",
              "      <td>-0.181123</td>\n",
              "      <td>-0.064700</td>\n",
              "      <td>0.101403</td>\n",
              "      <td>0.040688</td>\n",
              "      <td>0.551398</td>\n",
              "      <td>0.070826</td>\n",
              "      <td>-0.116083</td>\n",
              "      <td>0.271802</td>\n",
              "      <td>-0.312774</td>\n",
              "      <td>0.167951</td>\n",
              "      <td>0.077226</td>\n",
              "      <td>0.206577</td>\n",
              "      <td>0.349563</td>\n",
              "      <td>0.465750</td>\n",
              "      <td>0.617657</td>\n",
              "      <td>...</td>\n",
              "      <td>-0.373169</td>\n",
              "      <td>0.231144</td>\n",
              "      <td>0.168584</td>\n",
              "      <td>-0.200773</td>\n",
              "      <td>-0.087082</td>\n",
              "      <td>-0.143997</td>\n",
              "      <td>0.113722</td>\n",
              "      <td>-0.152569</td>\n",
              "      <td>0.056071</td>\n",
              "      <td>-0.191464</td>\n",
              "      <td>-0.314181</td>\n",
              "      <td>0.044217</td>\n",
              "      <td>-0.012398</td>\n",
              "      <td>-0.152131</td>\n",
              "      <td>-0.303886</td>\n",
              "      <td>0.227813</td>\n",
              "      <td>0.002255</td>\n",
              "      <td>0.309717</td>\n",
              "      <td>-0.190364</td>\n",
              "      <td>0.186217</td>\n",
              "      <td>0.572211</td>\n",
              "      <td>-0.094748</td>\n",
              "      <td>0.228616</td>\n",
              "      <td>0.117155</td>\n",
              "      <td>0.412889</td>\n",
              "      <td>-0.045846</td>\n",
              "      <td>-0.356627</td>\n",
              "      <td>0.031634</td>\n",
              "      <td>0.374091</td>\n",
              "      <td>0.295170</td>\n",
              "      <td>0.222816</td>\n",
              "      <td>-0.165044</td>\n",
              "      <td>0.247229</td>\n",
              "      <td>-0.573572</td>\n",
              "      <td>-0.135302</td>\n",
              "      <td>0.215148</td>\n",
              "      <td>-0.134988</td>\n",
              "      <td>0.035388</td>\n",
              "      <td>-0.045371</td>\n",
              "      <td>-0.062465</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5227 rows × 768 columns</p>\n",
              "</div>"
            ],
            "text/plain": [
              "      BERTEmbed_0  BERTEmbed_1  ...  BERTEmbed_766  BERTEmbed_767\n",
              "0        0.297946     0.203994  ...      -0.035265      -0.057258\n",
              "1        0.295606     0.204244  ...      -0.042629      -0.063909\n",
              "2        0.301102     0.203472  ...      -0.050423      -0.071233\n",
              "3        0.298717     0.208594  ...      -0.031238      -0.058959\n",
              "4        0.301026     0.203712  ...      -0.040579      -0.059467\n",
              "...           ...          ...  ...            ...            ...\n",
              "5222    -0.161756    -0.448114  ...       0.022575      -0.495942\n",
              "5223    -0.593530    -0.523108  ...      -0.006910      -0.514361\n",
              "5224     0.300492     0.191589  ...      -0.031499      -0.057095\n",
              "5225    -0.004848    -0.034025  ...      -0.148900      -0.236210\n",
              "5226     0.303914     0.201205  ...      -0.045371      -0.062465\n",
              "\n",
              "[5227 rows x 768 columns]"
            ]
          },
          "execution_count": 20,
          "metadata": {
            "tags": []
          },
          "output_type": "execute_result"
        }
      ],
      "source": [
        "df_result"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Z64V1gxaD7_u"
      },
      "outputs": [],
      "source": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [
        "v8u0ebtirRci",
        "vtIMlTiMUFzS",
        "LYT-KkTor3QE",
        "zNAzuZdm-5eY",
        "F08yfWtRAzty"
      ],
      "include_colab_link": true,
      "name": "CLF_BERT.ipynb",
      "provenance": [],
      "toc_visible": true
    },
    "kernelspec": {
      "display_name": "Python 3.8.8 64-bit (conda)",
      "name": "python388jvsc74a57bd0b3e779290c3971bcb91630500d66c3c3cecd721489b4b277b4ac4fef67ef773c"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.8"
    },
    "metadata": {
      "interpreter": {
        "hash": "b3e779290c3971bcb91630500d66c3c3cecd721489b4b277b4ac4fef67ef773c"
      }
    },
    "orig_nbformat": 2,
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "08b32242d368498daa7828b7afeb2a9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "0a75ab30a15140db902caf400674c115": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "0d25491d17b242b38c658b3ad668de1e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0dcbf81848f9482ea0bcb415241d13bb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1a43d70de32d4def9a1fe6966d96650b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1e80b1e80add4325a6750caeab4c54b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0dcbf81848f9482ea0bcb415241d13bb",
            "max": 542529064,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b19a9345a08d4154aaa3aec5d34b7a13",
            "value": 542529064
          }
        },
        "283c37bb7a4c42c2853f77a0e82c9a0e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2c22c266bd5c4ccea6b245e4b990971c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7f62faaa6cf947f48cde0b97d5fff4d7",
              "IPY_MODEL_fb8c67a56d8b4c4986c6d815cb4c6b3e"
            ],
            "layout": "IPY_MODEL_be000548455044719249d519495e514a"
          }
        },
        "3b53ece024be4582879ecfb989e30b75": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5ae373acd0d6406782cdd393667a5eae": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_e956c2c49a7648e1911ff94c3c403c9b",
              "IPY_MODEL_ab141a8f69d54292abcd5fb01104fdd0"
            ],
            "layout": "IPY_MODEL_0d25491d17b242b38c658b3ad668de1e"
          }
        },
        "68ddd0f509094751995c794a0dd9d3b0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b0e7b89c3e314dec92e6d355c5b8f99e",
            "placeholder": "​",
            "style": "IPY_MODEL_6b12e2ebcfcf4088a81abd6839a1fa06",
            "value": " 843k/843k [00:00&lt;00:00, 1.27MB/s]"
          }
        },
        "6b12e2ebcfcf4088a81abd6839a1fa06": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6f2c1baae7c7462091f69527d989a792": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7f62faaa6cf947f48cde0b97d5fff4d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_dd7dc5791f9142138fa0265c49d4be74",
            "max": 558,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_9eb0aaa6736e43c58cc44ce1504e5366",
            "value": 558
          }
        },
        "9e30225cf2364a548c06b6eecc8727df": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9eb0aaa6736e43c58cc44ce1504e5366": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "ab141a8f69d54292abcd5fb01104fdd0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6f2c1baae7c7462091f69527d989a792",
            "placeholder": "​",
            "style": "IPY_MODEL_1a43d70de32d4def9a1fe6966d96650b",
            "value": " 1.08M/1.08M [00:00&lt;00:00, 4.02MB/s]"
          }
        },
        "acbcd81cab3b42539ac733b7c19098c1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b0e7b89c3e314dec92e6d355c5b8f99e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b19a9345a08d4154aaa3aec5d34b7a13": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": "initial"
          }
        },
        "b19ffd9730f54aa2aa484ba1e6e418d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be000548455044719249d519495e514a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "be973816203644c69f9ba660c08b373c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_c4a9e8b9afe5416b8e229a4f078dc238",
              "IPY_MODEL_68ddd0f509094751995c794a0dd9d3b0"
            ],
            "layout": "IPY_MODEL_b19ffd9730f54aa2aa484ba1e6e418d9"
          }
        },
        "c070a13922844983ad5ae16730544deb": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c4a9e8b9afe5416b8e229a4f078dc238": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_acbcd81cab3b42539ac733b7c19098c1",
            "max": 843438,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_0a75ab30a15140db902caf400674c115",
            "value": 843438
          }
        },
        "cdf8aa309771498786713193c162b7c5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d4ed58d884a84191a2837daa47cba485": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1e80b1e80add4325a6750caeab4c54b6",
              "IPY_MODEL_d82470e0c93e40bba3a66cbe425393bb"
            ],
            "layout": "IPY_MODEL_3b53ece024be4582879ecfb989e30b75"
          }
        },
        "d82470e0c93e40bba3a66cbe425393bb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c070a13922844983ad5ae16730544deb",
            "placeholder": "​",
            "style": "IPY_MODEL_9e30225cf2364a548c06b6eecc8727df",
            "value": " 543M/543M [00:12&lt;00:00, 41.8MB/s]"
          }
        },
        "dcc1de2c1c1948e98c097472cea17686": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dd7dc5791f9142138fa0265c49d4be74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e956c2c49a7648e1911ff94c3c403c9b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "Downloading: 100%",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cdf8aa309771498786713193c162b7c5",
            "max": 1078931,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_08b32242d368498daa7828b7afeb2a9d",
            "value": 1078931
          }
        },
        "fb8c67a56d8b4c4986c6d815cb4c6b3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_283c37bb7a4c42c2853f77a0e82c9a0e",
            "placeholder": "​",
            "style": "IPY_MODEL_dcc1de2c1c1948e98c097472cea17686",
            "value": " 558/558 [00:00&lt;00:00, 2.38kB/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}