Bilal_et_al_Baseline.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/toby-p/w266-final-project/blob/main/Bilal_et_al_Baseline.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "YXAL6gpkijkz"
      },
      "outputs": [],
      "source": [
        "!pip install transformers --quiet"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "Kd0xo5RHVbg-"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.model_selection import train_test_split\n",
        "import tensorflow as tf\n",
        "from tensorflow import keras\n",
        "from transformers import BertTokenizer, TFBertForSequenceClassification "
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 70
        },
        "id": "Ms69Utx-0kHS",
        "outputId": "39098ae8-2699-4b0b-caae-f768147c3a6b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n",
            "Created checkpoint dir: /content/gdrive/MyDrive/models/test_model/checkpoints/2022_07_17__23_02_17\n"
          ]
        },
        {
          "data": {
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            },
            "text/plain": [
              "'/content/gdrive/MyDrive/models/test_model/checkpoints/2022_07_17__23_02_17'"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# Code for helping save models to GDrive after training:\n",
        "\n",
        "import datetime\n",
        "import os\n",
        "\n",
        "from google.colab import drive\n",
        "\n",
        "# Mount Google Drive:\n",
        "drive.mount(\"/content/gdrive\")\n",
        "\n",
        "# Directory where models will be stored in GDrive:\n",
        "MODEL_DIR = \"/content/gdrive/MyDrive/models\"\n",
        "\n",
        "# Make the directories for storing results if they don't exist yet:\n",
        "if not os.path.exists(MODEL_DIR):\n",
        "    os.mkdir(MODEL_DIR)\n",
        "\n",
        "\n",
        "def gdrive_save_dir(*subdir: str, model_name: str = \"test_model\"): \n",
        "    \"\"\"Create timestamped directory in GDrive for storing checkpoints or models.\n",
        "    \n",
        "    Args:\n",
        "        subdir: optional subdirectories of the main model directory\n",
        "            (e.g. `checkpoints`, `final_model`, etc.)\n",
        "        model_name: main name for directory specifying the model being saved.\n",
        "    \"\"\"\n",
        "    model_dir = f\"{MODEL_DIR}/{model_name}\"\n",
        "    if not os.path.exists(model_dir):\n",
        "        os.mkdir(model_dir)\n",
        "    for s in subdir:\n",
        "        model_dir = f\"{model_dir}/{s}\"\n",
        "        if not os.path.exists(model_dir):\n",
        "            os.mkdir(model_dir)\n",
        "    now = datetime.datetime.now()\n",
        "    now_str = now.strftime(\"%Y_%m_%d__%H_%M_%S\")\n",
        "    dir_path = f\"{model_dir}/{now_str}\"\n",
        "    os.mkdir(dir_path)\n",
        "    print(f\"Created checkpoint dir: {dir_path}\")\n",
        "    return dir_path\n",
        "\n",
        "\n",
        "gdrive_save_dir(\"checkpoints\", model_name = \"test_model\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JVa_1CPEIMv0"
      },
      "source": [
        "## Create train and test data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "DDJUbITGk83S"
      },
      "outputs": [],
      "source": [
        "# Using the data downloaded from here: https://sites.google.com/view/review-helpfulness-prediction/datasets\n",
        "# Saved publicly in Google Drive.\n",
        "\n",
        "def get_google_drive_download_url(raw_url: str):\n",
        "    return \"https://drive.google.com/uc?id=\" + raw_url.split(\"/\")[-2]\n",
        "\n",
        "\n",
        "train_url = \"https://drive.google.com/file/d/1i54O_JSAVtvP5ivor-ARJRkwSoBFdit1/view?usp=sharing\"\n",
        "test_url = \"https://drive.google.com/file/d/1boRdmasHB6JZDNBrlt6MRB1pUVnxxY-6/view?usp=sharing\"\n",
        "\n",
        "bilal_train = pd.read_csv(get_google_drive_download_url(train_url), encoding=\"latin1\")\n",
        "bilal_test = pd.read_csv(get_google_drive_download_url(test_url), encoding=\"latin1\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "-IDQWfD-oet_",
        "outputId": "0ffee98f-cc62-40df-b1cd-258ff2d45308"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-a921aa05-2b6d-4d81-9dee-78cb936de9bf\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>7995</th>\n",
              "      <td>89260</td>\n",
              "      <td>Easy access off the 101 lots of parking in the...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7996</th>\n",
              "      <td>62116</td>\n",
              "      <td>Meh. I went in for some accessories and a part...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7997</th>\n",
              "      <td>11115</td>\n",
              "      <td>Worst customer service ever. I called the stor...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7998</th>\n",
              "      <td>11885</td>\n",
              "      <td>I had my Canon Rebel T1i repaired after I drop...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>7999</th>\n",
              "      <td>53295</td>\n",
              "      <td>Great store a little short on boys youth sizes...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-a921aa05-2b6d-4d81-9dee-78cb936de9bf')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-a921aa05-2b6d-4d81-9dee-78cb936de9bf button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-a921aa05-2b6d-4d81-9dee-78cb936de9bf');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         id                                           sentence  label\n",
              "7995  89260  Easy access off the 101 lots of parking in the...      0\n",
              "7996  62116  Meh. I went in for some accessories and a part...      0\n",
              "7997  11115  Worst customer service ever. I called the stor...      0\n",
              "7998  11885  I had my Canon Rebel T1i repaired after I drop...      0\n",
              "7999  53295  Great store a little short on boys youth sizes...      0"
            ]
          },
          "execution_count": 5,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "bilal_train.tail()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "UbGZScrEof99",
        "outputId": "b720d804-0d83-45e3-8a33-5fd91f392d8e"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-e76189e3-082b-4fe0-8cf1-c0f3dfab54d3\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>sentence</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>1995</th>\n",
              "      <td>63354</td>\n",
              "      <td>Big sale this week. All sort of little gadets ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1996</th>\n",
              "      <td>45423</td>\n",
              "      <td>The new owner and management are great. I didn...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1997</th>\n",
              "      <td>12024</td>\n",
              "      <td>Came here to check out their Patio Furniture. ...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1998</th>\n",
              "      <td>89218</td>\n",
              "      <td>I brought in a flash drive with a 3-page docum...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1999</th>\n",
              "      <td>45672</td>\n",
              "      <td>Super helpful. Taught me exactly how to gel st...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e76189e3-082b-4fe0-8cf1-c0f3dfab54d3')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e76189e3-082b-4fe0-8cf1-c0f3dfab54d3 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e76189e3-082b-4fe0-8cf1-c0f3dfab54d3');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ],
            "text/plain": [
              "         id                                           sentence  label\n",
              "1995  63354  Big sale this week. All sort of little gadets ...      0\n",
              "1996  45423  The new owner and management are great. I didn...      0\n",
              "1997  12024  Came here to check out their Patio Furniture. ...      0\n",
              "1998  89218  I brought in a flash drive with a 3-page docum...      0\n",
              "1999  45672  Super helpful. Taught me exactly how to gel st...      0"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "bilal_test.tail()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "LV0EHztAkLZ_",
        "outputId": "237d5c68-bde5-4b3b-ca6a-a1ac1294df23"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train class balance:  0.5\n",
            "Test class balance:  0.5\n"
          ]
        }
      ],
      "source": [
        "# See if the classes are even:\n",
        "print(\"Train class balance: \", bilal_train[\"label\"].mean())\n",
        "print(\"Test class balance: \", bilal_test[\"label\"].mean())"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "vvzKO4Wj4f6c"
      },
      "outputs": [],
      "source": [
        "x_train_full = bilal_train[\"sentence\"]\n",
        "y_train_full = bilal_train[\"label\"]\n",
        "x_test = bilal_test[\"sentence\"]\n",
        "y_test = bilal_test[\"label\"]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "GpZ1130U4yLK",
        "outputId": "9c82bde2-a065-4453-e7ad-94781c2a0e64"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Shape X_train: (7200,)\n",
            "Shape X_valid: (800,)\n",
            "Shape y_train: (7200,)\n",
            "Shape y_val: (800,)\n"
          ]
        }
      ],
      "source": [
        "# Split train into 90-10 split for train-validation as per the paper:\n",
        "x_train, x_val, y_train, y_val = train_test_split(x_train_full, y_train_full, test_size=0.1)\n",
        "\n",
        "print(f\"Shape X_train: {x_train.shape}\")\n",
        "print(f\"Shape X_valid: {x_val.shape}\")\n",
        "print(f\"Shape y_train: {y_train.shape}\")\n",
        "print(f\"Shape y_val: {y_val.shape}\")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "HcyU1oVWakMM"
      },
      "source": [
        "## Tokenize inputs"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true
        },
        "id": "KYbFcFevWm_2"
      },
      "outputs": [],
      "source": [
        "# Using BERT base uncased tokenizer as per the paper:\n",
        "bert_tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
        "\n",
        "# Use sequence length 320, which achieved best accuracy and F1-score of all sequence lengths tried in the paper:\n",
        "# https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/4\n",
        "max_length = 320\n",
        "\n",
        "train_encodings = bert_tokenizer(\n",
        "    list(x_train.values), \n",
        "    max_length=max_length,\n",
        "    truncation=True,\n",
        "    padding='max_length', \n",
        "    return_tensors='tf'\n",
        ")\n",
        "\n",
        "valid_encodings = bert_tokenizer(\n",
        "    list(x_val.values), \n",
        "    max_length=max_length,\n",
        "    truncation=True,\n",
        "    padding='max_length', \n",
        "    return_tensors='tf'\n",
        ")\n",
        "\n",
        "test_encodings = bert_tokenizer(\n",
        "    list(x_test.values), \n",
        "    max_length=max_length,\n",
        "    truncation=True,\n",
        "    padding='max_length', \n",
        "    return_tensors='tf'\n",
        ")"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "qQ045x2mH4la"
      },
      "source": [
        "## Model 1: Baseline model with unfrozen classification layers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "rFBftqxCZF-v"
      },
      "outputs": [],
      "source": [
        "\n",
        "MODEL_NAME = \"bilal_classifier_layers_only\"\n",
        "\n",
        "\n",
        "def bilal_classifier_layers_only():\n",
        "    \"\"\"Create a BERT model using the model and parameters specified in the Bilal paper:\n",
        "    https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "\n",
        "        - model: TFBertForSequenceClassification\n",
        "        - learning rate: 2e-5\n",
        "        - epsilon: 1e-8\n",
        "    \"\"\"\n",
        "    # Using the TFBertForSequenceClassification as specified in the paper:\n",
        "    bert_model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "\n",
        "    # Freeze all layers except the last 4 which are the pooled classification layers:\n",
        "    untrainable = [w.name for w in bert_model.weights[:-4]]\n",
        "    trainable = [w.name for w in bert_model.weights[-4:]]\n",
        "\n",
        "    for w in bert_model.weights:\n",
        "        if w.name in untrainable:\n",
        "            w._trainable = False\n",
        "        elif w.name in trainable:\n",
        "            w._trainable = True\n",
        "\n",
        "    # Compile the model:\n",
        "    bert_model.compile(\n",
        "        optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5,epsilon=1e-08),\n",
        "        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
        "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")]\n",
        "    )\n",
        "\n",
        "    return bert_model\n",
        "\n",
        "\n",
        "model = bilal_classifier_layers_only()\n",
        "print(model.summary())\n",
        "\n",
        "# Train the model using the specifications from the paper: https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "# -- epochs = 4\n",
        "# -- batch_size = 32\n",
        "\n",
        "# Create directory for storing checkpoints after each epoch:\n",
        "checkpoint_dir = gdrive_save_dir(\"checkpoints\", model_name = MODEL_NAME)\n",
        "checkpoint_path = checkpoint_dir + \"/cp-{epoch:04d}.ckpt\"\n",
        "\n",
        "# Create a callback that saves the model's weights:\n",
        "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_path,\n",
        "    save_weights_only=True,\n",
        "    verbose=1)\n",
        "\n",
        "# Fit the model saving weights every epoch:\n",
        "history = model.fit(\n",
        "    [train_encodings.input_ids, train_encodings.token_type_ids, train_encodings.attention_mask], \n",
        "    y_train.values,\n",
        "    validation_data=(\n",
        "        [valid_encodings.input_ids, valid_encodings.token_type_ids, valid_encodings.attention_mask], \n",
        "        y_val.values\n",
        "        ),\n",
        "    batch_size=32, \n",
        "    epochs=4,\n",
        "    callbacks=[cp_callback]\n",
        ")\n",
        "\n",
        "# Save the entire model to GDrive:\n",
        "model_dir = gdrive_save_dir(\"full_model\", model_name = MODEL_NAME)\n",
        "model.save(model_dir)\n",
        "\n",
        "# Save scores on the test set:\n",
        "test_score = model.evaluate([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask], y_test)\n",
        "print(\"Test loss:\", test_score[0])\n",
        "print(\"Test accuracy:\", test_score[1])\n",
        "score_fp = os.path.join(model_dir, \"test_score.txt\")\n",
        "with open(score_fp, \"w\") as f:\n",
        "    f.write(f\"Test loss = {test_score[0]}\\n\")\n",
        "    f.write(f\"Test accuracy = {test_score[1]}\\n\")\n",
        "\n",
        "# Save predictions and classification_report:\n",
        "predictions = model.predict([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask])\n",
        "preds_fp = os.path.join(model_dir, \"test_predictions.csv\")\n",
        "pred_df = pd.DataFrame(predictions.to_tuple()[0], columns=[\"pred_prob_0\", \"pred_prob_1\"])\n",
        "pred_df[\"yhat\"] = pred_df[[\"pred_prob_0\", \"pred_prob_1\"]].values.argmax(1)\n",
        "pred_df[\"y\"] = y_test\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 1) & (pred_df[\"y\"] == 1), \"tp\", None)\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 0) & (pred_df[\"y\"] == 0), \"tn\", pred_df[\"category\"])\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 1) & (pred_df[\"y\"] == 0), \"fp\", pred_df[\"category\"])\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 0) & (pred_df[\"y\"] == 1), \"fn\", pred_df[\"category\"])\n",
        "pred_df.to_csv(preds_fp, encoding=\"utf-8\", index=False)\n",
        "report = classification_report(y_test, pred_df[\"yhat\"])\n",
        "report_fp = os.path.join(model_dir, \"classification_report.txt\")\n",
        "with open(report_fp, \"w\") as f:\n",
        "    for line in report.split(\"\\n\"):\n",
        "        f.write(f\"{line}\\n\")\n",
        "print(f\"{MODEL_NAME} - test set results\")\n",
        "print(report)"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "XtosRoO6i-Nq"
      },
      "source": [
        "## Model 2: removing dropout\n",
        "\n",
        "Model seems to be better at not overfitting than in the paper. Since the paper didn't mention dropout, we try the same model again but with no dropout layer."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "eTLFiHZbg5p7"
      },
      "outputs": [],
      "source": [
        "def bilal_bert_model_no_dropout():\n",
        "    \"\"\"Create a BERT model using the model and parameters specified in the Bilal paper:\n",
        "    https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "\n",
        "        - model: TFBertForSequenceClassification\n",
        "        - learning rate: 2e-5\n",
        "        - epsilon: 1e-8\n",
        "\n",
        "    This time removes dropout from classification layer to see if it better matches paper's results.\n",
        "    \"\"\"\n",
        "    # Using the TFBertForSequenceClassification as specified in the paper:\n",
        "    bert_model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2, hidden_dropout_prob=0)\n",
        "\n",
        "    # Freeze all layers except the last 4 which are the pooled classification layers:\n",
        "    untrainable = [w.name for w in bert_model.weights[:-4]]\n",
        "    trainable = [w.name for w in bert_model.weights[-4:]]\n",
        "\n",
        "    for w in bert_model.weights:\n",
        "        if w.name in untrainable:\n",
        "            w._trainable = False\n",
        "        elif w.name in trainable:\n",
        "            w._trainable = True\n",
        "\n",
        "    # Compile the model:\n",
        "    bert_model.compile(\n",
        "        optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5,epsilon=1e-08),\n",
        "        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
        "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")]\n",
        "    )\n",
        "\n",
        "    return bert_model"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 379,
          "referenced_widgets": [
            "df6c2176f53d4f84bc8ed41c7f50a30e",
            "d5efaed07bb5477bb017a7b5f3c2bcbd",
            "378723b00eb24af8b8ef0fa431d9e989",
            "8cb8d416840943459346544180cdf412",
            "3156d465a82c4415a9a8f926c8aadaa4",
            "2a255928c95b4e3d8333605d139c41a6",
            "ea19bad76ec344b8b22b681dcc0fa1c8",
            "ae96d56381864fbea945abfce52fd7a7",
            "402822ee8d584628a896f5b0632b40ec",
            "03a5a188f8a54d8b84b842016ab11981",
            "f0c04936bb394d19840792cf929f8f08"
          ]
        },
        "id": "Ts2wDgi2jQTQ",
        "outputId": "38d76e73-9711-4f6e-eaa7-fec819cf539d"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "df6c2176f53d4f84bc8ed41c7f50a30e",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "Downloading:   0%|          | 0.00/511M [00:00<?, ?B/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Model: \"tf_bert_for_sequence_classification\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " bert (TFBertMainLayer)      multiple                  109482240 \n",
            "                                                                 \n",
            " dropout_37 (Dropout)        multiple                  0         \n",
            "                                                                 \n",
            " classifier (Dense)          multiple                  1538      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 109,483,778\n",
            "Trainable params: 109,483,778\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n"
          ]
        }
      ],
      "source": [
        "model2 = bilal_bert_model_no_dropout()\n",
        "model2.summary()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ibfgGAmQiWsE",
        "outputId": "0b900d59-3b35-47f8-a442-112058e85d64"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1/4\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_for_sequence_classification/bert/embeddings/word_embeddings/weight:0', 'tf_bert_for_sequence_classification/bert/embeddings/token_type_embeddings/embeddings:0', 'tf_bert_for_sequence_classification/bert/embeddings/position_embeddings/embeddings:0', 'tf_bert_for_sequence_classification/bert/embeddings/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/embeddings/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/LayerNorm/beta:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_bert_for_sequence_classification/bert/embeddings/word_embeddings/weight:0', 'tf_bert_for_sequence_classification/bert/embeddings/token_type_embeddings/embeddings:0', 'tf_bert_for_sequence_classification/bert/embeddings/position_embeddings/embeddings:0', 'tf_bert_for_sequence_classification/bert/embeddings/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/embeddings/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._0/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._1/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._2/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._3/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._4/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._5/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._6/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._7/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._8/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._9/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._10/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/query/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/query/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/key/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/key/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/value/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/self/value/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/attention/output/LayerNorm/beta:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/intermediate/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/intermediate/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/dense/kernel:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/dense/bias:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/LayerNorm/gamma:0', 'tf_bert_for_sequence_classification/bert/encoder/layer_._11/output/LayerNorm/beta:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "225/225 [==============================] - 1276s 6s/step - loss: 0.6104 - accuracy: 0.6771 - val_loss: 0.6046 - val_accuracy: 0.6775\n",
            "Epoch 2/4\n",
            "225/225 [==============================] - 1283s 6s/step - loss: 0.5918 - accuracy: 0.6942 - val_loss: 0.6011 - val_accuracy: 0.6750\n",
            "Epoch 3/4\n",
            "225/225 [==============================] - 1665s 7s/step - loss: 0.5874 - accuracy: 0.6997 - val_loss: 0.5976 - val_accuracy: 0.6825\n",
            "Epoch 4/4\n",
            "225/225 [==============================] - 1658s 7s/step - loss: 0.5816 - accuracy: 0.7017 - val_loss: 0.5948 - val_accuracy: 0.6850\n"
          ]
        }
      ],
      "source": [
        "# Train the model using the specifications from the paper: https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "# -- epochs = 4\n",
        "# -- batch_size = 32\n",
        "\n",
        "history2 = model2.fit(\n",
        "    [train_encodings.input_ids, train_encodings.token_type_ids, train_encodings.attention_mask], \n",
        "    y_train.values,\n",
        "    validation_data=(\n",
        "        [valid_encodings.input_ids, valid_encodings.token_type_ids, valid_encodings.attention_mask], \n",
        "        y_val.values\n",
        "        ),\n",
        "    batch_size=32, \n",
        "    epochs=4\n",
        ")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "EjyJGqRNBvup",
        "outputId": "bcb4f5cb-e0ea-4d74-c1ba-d3787f160b50"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "63/63 [==============================] - 400s 6s/step - loss: 0.5945 - accuracy: 0.7015\n",
            "Test loss: 0.5944662690162659\n",
            "Test accuracy: 0.7014999985694885\n"
          ]
        }
      ],
      "source": [
        "score = model2.evaluate([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask], y_test)\n",
        "\n",
        "print(\"Test loss:\", score[0])\n",
        "print(\"Test accuracy:\", score[1])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "WLvOZIcpBvxj",
        "outputId": "da39678d-1261-4deb-eda4-6a00b1244bc5"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.71      0.67      0.69      1000\n",
            "           1       0.69      0.73      0.71      1000\n",
            "\n",
            "    accuracy                           0.70      2000\n",
            "   macro avg       0.70      0.70      0.70      2000\n",
            "weighted avg       0.70      0.70      0.70      2000\n",
            "\n"
          ]
        }
      ],
      "source": [
        "predictions = model2.predict([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask])\n",
        "preds = predictions.to_tuple()[0].argmax(1)\n",
        "print(classification_report(y_test, preds))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sL61_-AsltW6"
      },
      "source": [
        "## Model 3: Full finetuning (don't freeze any layers)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "23iFSFzEYB7P",
        "outputId": "23108167-6e69-4031-bcbf-60871ba5a430"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFBertForSequenceClassification.\n",
            "\n",
            "Some layers of TFBertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Model: \"tf_bert_for_sequence_classification\"\n",
            "_________________________________________________________________\n",
            " Layer (type)                Output Shape              Param #   \n",
            "=================================================================\n",
            " bert (TFBertMainLayer)      multiple                  109482240 \n",
            "                                                                 \n",
            " dropout_37 (Dropout)        multiple                  0         \n",
            "                                                                 \n",
            " classifier (Dense)          multiple                  1538      \n",
            "                                                                 \n",
            "=================================================================\n",
            "Total params: 109,483,778\n",
            "Trainable params: 109,483,778\n",
            "Non-trainable params: 0\n",
            "_________________________________________________________________\n",
            "None\n",
            "Created checkpoint dir: /content/gdrive/MyDrive/models/bilal_full_finetune/checkpoints/2022_07_17__23_02_59\n",
            "Epoch 1/4\n",
            "225/225 [==============================] - ETA: 0s - loss: 0.6103 - accuracy: 0.6840 \n",
            "Epoch 1: saving model to /content/gdrive/MyDrive/models/bilal_full_finetune/checkpoints/2022_07_17__23_02_59/cp-0001.ckpt\n",
            "225/225 [==============================] - 4560s 20s/step - loss: 0.6103 - accuracy: 0.6840 - val_loss: 0.5857 - val_accuracy: 0.7063\n",
            "Epoch 2/4\n",
            "225/225 [==============================] - ETA: 0s - loss: 0.5910 - accuracy: 0.6967 \n",
            "Epoch 2: saving model to /content/gdrive/MyDrive/models/bilal_full_finetune/checkpoints/2022_07_17__23_02_59/cp-0002.ckpt\n",
            "225/225 [==============================] - 4556s 20s/step - loss: 0.5910 - accuracy: 0.6967 - val_loss: 0.5754 - val_accuracy: 0.7100\n",
            "Epoch 3/4\n",
            "225/225 [==============================] - ETA: 0s - loss: 0.5610 - accuracy: 0.7188 \n",
            "Epoch 3: saving model to /content/gdrive/MyDrive/models/bilal_full_finetune/checkpoints/2022_07_17__23_02_59/cp-0003.ckpt\n",
            "225/225 [==============================] - 4518s 20s/step - loss: 0.5610 - accuracy: 0.7188 - val_loss: 0.5782 - val_accuracy: 0.7013\n",
            "Epoch 4/4\n",
            "225/225 [==============================] - ETA: 0s - loss: 0.4971 - accuracy: 0.7642 \n",
            "Epoch 4: saving model to /content/gdrive/MyDrive/models/bilal_full_finetune/checkpoints/2022_07_17__23_02_59/cp-0004.ckpt\n",
            "225/225 [==============================] - 4517s 20s/step - loss: 0.4971 - accuracy: 0.7642 - val_loss: 0.5744 - val_accuracy: 0.7125\n",
            "Created checkpoint dir: /content/gdrive/MyDrive/models/bilal_full_finetune/full_model/2022_07_18__04_05_30\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "WARNING:absl:Found untraced functions such as embeddings_layer_call_fn, embeddings_layer_call_and_return_conditional_losses, encoder_layer_call_fn, encoder_layer_call_and_return_conditional_losses, pooler_layer_call_fn while saving (showing 5 of 420). These functions will not be directly callable after loading.\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "INFO:tensorflow:Assets written to: /content/gdrive/MyDrive/models/bilal_full_finetune/full_model/2022_07_18__04_05_30/assets\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "INFO:tensorflow:Assets written to: /content/gdrive/MyDrive/models/bilal_full_finetune/full_model/2022_07_18__04_05_30/assets\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "63/63 [==============================] - 367s 6s/step - loss: 0.5919 - accuracy: 0.6955\n",
            "Test loss: 0.5919166803359985\n",
            "Test accuracy: 0.6955000162124634\n",
            "bilal_full_finetune - test set results\n",
            "              precision    recall  f1-score   support\n",
            "\n",
            "           0       0.69      0.71      0.70      1000\n",
            "           1       0.70      0.68      0.69      1000\n",
            "\n",
            "    accuracy                           0.70      2000\n",
            "   macro avg       0.70      0.70      0.70      2000\n",
            "weighted avg       0.70      0.70      0.70      2000\n",
            "\n"
          ]
        }
      ],
      "source": [
        "\n",
        "MODEL_NAME = \"bilal_full_finetune\"\n",
        "\n",
        "\n",
        "def bilal_full_finetune():\n",
        "    \"\"\"Create a BERT model using the model and parameters specified in the Bilal paper:\n",
        "    https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "\n",
        "        - model: TFBertForSequenceClassification\n",
        "        - learning rate: 2e-5\n",
        "        - epsilon: 1e-8\n",
        "    \"\"\"\n",
        "    # Using the TFBertForSequenceClassification as specified in the paper:\n",
        "    bert_model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
        "\n",
        "    # Don't freeze any layers:\n",
        "    untrainable = []\n",
        "    trainable = [w.name for w in bert_model.weights]\n",
        "\n",
        "    for w in bert_model.weights:\n",
        "        if w.name in untrainable:\n",
        "            w._trainable = False\n",
        "        elif w.name in trainable:\n",
        "            w._trainable = True\n",
        "\n",
        "    # Compile the model:\n",
        "    bert_model.compile(\n",
        "        optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5,epsilon=1e-08),\n",
        "        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
        "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")]\n",
        "    )\n",
        "\n",
        "    return bert_model\n",
        "\n",
        "\n",
        "model = bilal_full_finetune()\n",
        "print(model.summary())\n",
        "\n",
        "# Train the model using the specifications from the paper: https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
        "# -- epochs = 4\n",
        "# -- batch_size = 32\n",
        "\n",
        "# Create directory for storing checkpoints after each epoch:\n",
        "checkpoint_dir = gdrive_save_dir(\"checkpoints\", model_name = MODEL_NAME)\n",
        "checkpoint_path = checkpoint_dir + \"/cp-{epoch:04d}.ckpt\"\n",
        "\n",
        "# Create a callback that saves the model's weights:\n",
        "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n",
        "    filepath=checkpoint_path,\n",
        "    save_weights_only=True,\n",
        "    verbose=1)\n",
        "\n",
        "# Fit the model saving weights every epoch:\n",
        "history = model.fit(\n",
        "    [train_encodings.input_ids, train_encodings.token_type_ids, train_encodings.attention_mask], \n",
        "    y_train.values,\n",
        "    validation_data=(\n",
        "        [valid_encodings.input_ids, valid_encodings.token_type_ids, valid_encodings.attention_mask], \n",
        "        y_val.values\n",
        "        ),\n",
        "    batch_size=32, \n",
        "    epochs=4,\n",
        "    callbacks=[cp_callback]\n",
        ")\n",
        "\n",
        "# Save the entire model to GDrive:\n",
        "model_dir = gdrive_save_dir(\"full_model\", model_name = MODEL_NAME)\n",
        "model.save(model_dir)\n",
        "\n",
        "# Save scores on the test set:\n",
        "test_score = model.evaluate([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask], y_test)\n",
        "print(\"Test loss:\", test_score[0])\n",
        "print(\"Test accuracy:\", test_score[1])\n",
        "score_fp = os.path.join(model_dir, \"test_score.txt\")\n",
        "with open(score_fp, \"w\") as f:\n",
        "    f.write(f\"Test loss = {test_score[0]}\\n\")\n",
        "    f.write(f\"Test accuracy = {test_score[1]}\\n\")\n",
        "\n",
        "# Save predictions and classification_report:\n",
        "predictions = model.predict([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask])\n",
        "preds_fp = os.path.join(model_dir, \"test_predictions.csv\")\n",
        "pred_df = pd.DataFrame(predictions.to_tuple()[0], columns=[\"pred_prob_0\", \"pred_prob_1\"])\n",
        "pred_df[\"yhat\"] = pred_df[[\"pred_prob_0\", \"pred_prob_1\"]].values.argmax(1)\n",
        "pred_df[\"y\"] = y_test\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 1) & (pred_df[\"y\"] == 1), \"tp\", None)\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 0) & (pred_df[\"y\"] == 0), \"tn\", pred_df[\"category\"])\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 1) & (pred_df[\"y\"] == 0), \"fp\", pred_df[\"category\"])\n",
        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 0) & (pred_df[\"y\"] == 1), \"fn\", pred_df[\"category\"])\n",
        "pred_df.to_csv(preds_fp, encoding=\"utf-8\", index=False)\n",
        "report = classification_report(y_test, pred_df[\"yhat\"])\n",
        "report_fp = os.path.join(model_dir, \"classification_report.txt\")\n",
        "with open(report_fp, \"w\") as f:\n",
        "    for line in report.split(\"\\n\"):\n",
        "        f.write(f\"{line}\\n\")\n",
        "print(f\"{MODEL_NAME} - test set results\")\n",
        "print(report)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2j2lhg8ggkY6"
      },
      "outputs": [],
      "source": [
        ""
      ]
    }
  ],
  "metadata": {
    "accelerator": "TPU",
    "colab": {
      "background_execution": "on",
      "collapsed_sections": [],
      "machine_shape": "hm",
      "name": "Bilal_et_al_Baseline.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "03a5a188f8a54d8b84b842016ab11981": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2a255928c95b4e3d8333605d139c41a6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3156d465a82c4415a9a8f926c8aadaa4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "378723b00eb24af8b8ef0fa431d9e989": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ae96d56381864fbea945abfce52fd7a7",
            "max": 536063208,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_402822ee8d584628a896f5b0632b40ec",
            "value": 536063208
          }
        },
        "402822ee8d584628a896f5b0632b40ec": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "8cb8d416840943459346544180cdf412": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_03a5a188f8a54d8b84b842016ab11981",
            "placeholder": "​",
            "style": "IPY_MODEL_f0c04936bb394d19840792cf929f8f08",
            "value": " 511M/511M [00:09&lt;00:00, 59.8MB/s]"
          }
        },
        "ae96d56381864fbea945abfce52fd7a7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5efaed07bb5477bb017a7b5f3c2bcbd": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2a255928c95b4e3d8333605d139c41a6",
            "placeholder": "​",
            "style": "IPY_MODEL_ea19bad76ec344b8b22b681dcc0fa1c8",
            "value": "Downloading: 100%"
          }
        },
        "df6c2176f53d4f84bc8ed41c7f50a30e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_d5efaed07bb5477bb017a7b5f3c2bcbd",
              "IPY_MODEL_378723b00eb24af8b8ef0fa431d9e989",
              "IPY_MODEL_8cb8d416840943459346544180cdf412"
            ],
            "layout": "IPY_MODEL_3156d465a82c4415a9a8f926c8aadaa4"
          }
        },
        "ea19bad76ec344b8b22b681dcc0fa1c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f0c04936bb394d19840792cf929f8f08": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}