From c443417686471c2395e8fd08a2d03f06b1c2d3a8 Mon Sep 17 00:00:00 2001
From: toby petty <toby.b.petty@gmail.com>
Date: Mon, 18 Jul 2022 22:21:25 -0400
Subject: [PATCH] Adding notebook for transfer learning piece.

---
 Amazon_product_data_transfer_learning.ipynb | 1877 +++++++++++++++++++
 1 file changed, 1877 insertions(+)
 create mode 100644 Amazon_product_data_transfer_learning.ipynb

diff --git a/Amazon_product_data_transfer_learning.ipynb b/Amazon_product_data_transfer_learning.ipynb
new file mode 100644
index 0000000..5fb44e4
--- /dev/null
+++ b/Amazon_product_data_transfer_learning.ipynb
@@ -0,0 +1,1877 @@
+{
+  "cells": [
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "view-in-github",
+        "colab_type": "text"
+      },
+      "source": [
+        "<a href=\"https://colab.research.google.com/github/toby-p/w266-final-project/blob/main/Amazon_product_data_transfer_learning.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "YXAL6gpkijkz",
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "outputId": "3ebd7658-10b0-4ba8-d5c4-99fb95044f4b"
+      },
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "\u001b[K     |████████████████████████████████| 4.4 MB 5.1 MB/s \n",
+            "\u001b[K     |████████████████████████████████| 6.6 MB 69.6 MB/s \n",
+            "\u001b[K     |████████████████████████████████| 596 kB 78.2 MB/s \n",
+            "\u001b[K     |████████████████████████████████| 101 kB 10.5 MB/s \n",
+            "\u001b[?25h"
+          ]
+        }
+      ],
+      "source": [
+        "!pip install transformers --quiet"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "Kd0xo5RHVbg-"
+      },
+      "outputs": [],
+      "source": [
+        "import pandas as pd\n",
+        "from sklearn.metrics import classification_report\n",
+        "from sklearn.model_selection import train_test_split\n",
+        "import tensorflow as tf\n",
+        "from tensorflow import keras\n",
+        "from transformers import BertTokenizer, TFBertForSequenceClassification "
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Code for helping save models to GDrive after training:\n",
+        "\n",
+        "import datetime\n",
+        "import os\n",
+        "\n",
+        "from google.colab import drive\n",
+        "\n",
+        "# Mount Google Drive:\n",
+        "drive.mount(\"/content/gdrive\")\n",
+        "\n",
+        "# Directory where models will be stored in GDrive:\n",
+        "MODEL_DIR = \"/content/gdrive/MyDrive/models\"\n",
+        "\n",
+        "# Make the directories for storing results if they don't exist yet:\n",
+        "if not os.path.exists(MODEL_DIR):\n",
+        "    os.mkdir(MODEL_DIR)\n",
+        "\n",
+        "\n",
+        "def gdrive_save_dir(*subdir: str, model_name: str = \"test_model\"): \n",
+        "    \"\"\"Create timestamped directory in GDrive for storing checkpoints or models.\n",
+        "    \n",
+        "    Args:\n",
+        "        subdir: optional subdirectories of the main model directory\n",
+        "            (e.g. `checkpoints`, `final_model`, etc.)\n",
+        "        model_name: main name for directory specifying the model being saved.\n",
+        "    \"\"\"\n",
+        "    model_dir = f\"{MODEL_DIR}/{model_name}\"\n",
+        "    if not os.path.exists(model_dir):\n",
+        "        os.mkdir(model_dir)\n",
+        "    for s in subdir:\n",
+        "        model_dir = f\"{model_dir}/{s}\"\n",
+        "        if not os.path.exists(model_dir):\n",
+        "            os.mkdir(model_dir)\n",
+        "    now = datetime.datetime.now()\n",
+        "    now_str = now.strftime(\"%Y_%m_%d__%H_%M_%S\")\n",
+        "    dir_path = f\"{model_dir}/{now_str}\"\n",
+        "    os.mkdir(dir_path)\n",
+        "    print(f\"Created checkpoint dir: {dir_path}\")\n",
+        "    return dir_path\n",
+        "\n",
+        "\n",
+        "gdrive_save_dir(\"checkpoints\", model_name = \"test_model\")"
+      ],
+      "metadata": {
+        "id": "Ms69Utx-0kHS",
+        "outputId": "4e1d4e86-e6c1-49dd-c03f-46ac7bf770fa",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 70
+        }
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Mounted at /content/gdrive\n",
+            "Created checkpoint dir: /content/gdrive/MyDrive/models/test_model/checkpoints/2022_07_17__20_21_56\n"
+          ]
+        },
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "'/content/gdrive/MyDrive/models/test_model/checkpoints/2022_07_17__20_21_56'"
+            ],
+            "application/vnd.google.colaboratory.intrinsic+json": {
+              "type": "string"
+            }
+          },
+          "metadata": {},
+          "execution_count": 3
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "JVa_1CPEIMv0"
+      },
+      "source": [
+        "## Create train and test data"
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "# Using the datasets created in a separate notebook and saved to Github:\n",
+        "train_url = \"https://raw.githubusercontent.com/toby-p/w266-final-project/main/data/amazon/train.csv\"\n",
+        "test_url = \"https://raw.githubusercontent.com/toby-p/w266-final-project/main/data/amazon/test.csv\"\n",
+        "val_url = \"https://raw.githubusercontent.com/toby-p/w266-final-project/main/data/amazon/val.csv\"\n",
+        "\n",
+        "amazon_train = pd.read_csv(train_url, encoding=\"latin1\")\n",
+        "amazon_test = pd.read_csv(test_url, encoding=\"latin1\")\n",
+        "amazon_val = pd.read_csv(val_url, encoding=\"latin1\")"
+      ],
+      "metadata": {
+        "id": "DDJUbITGk83S"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "amazon_train.tail()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 206
+        },
+        "id": "-IDQWfD-oet_",
+        "outputId": "0c639ca9-acbf-4188-8a57-3aa906406465"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "         id                                           sentence  label\n",
+              "7995  89260  Easy access off the 101 lots of parking in the...      0\n",
+              "7996  62116  Meh. I went in for some accessories and a part...      0\n",
+              "7997  11115  Worst customer service ever. I called the stor...      0\n",
+              "7998  11885  I had my Canon Rebel T1i repaired after I drop...      0\n",
+              "7999  53295  Great store a little short on boys youth sizes...      0"
+            ],
+            "text/html": [
+              "\n",
+              "  <div id=\"df-32f1a583-4613-4255-bca3-f005286dd3e1\">\n",
+              "    <div class=\"colab-df-container\">\n",
+              "      <div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th></th>\n",
+              "      <th>id</th>\n",
+              "      <th>sentence</th>\n",
+              "      <th>label</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>7995</th>\n",
+              "      <td>89260</td>\n",
+              "      <td>Easy access off the 101 lots of parking in the...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>7996</th>\n",
+              "      <td>62116</td>\n",
+              "      <td>Meh. I went in for some accessories and a part...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>7997</th>\n",
+              "      <td>11115</td>\n",
+              "      <td>Worst customer service ever. I called the stor...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>7998</th>\n",
+              "      <td>11885</td>\n",
+              "      <td>I had my Canon Rebel T1i repaired after I drop...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>7999</th>\n",
+              "      <td>53295</td>\n",
+              "      <td>Great store a little short on boys youth sizes...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>\n",
+              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-32f1a583-4613-4255-bca3-f005286dd3e1')\"\n",
+              "              title=\"Convert this dataframe to an interactive table.\"\n",
+              "              style=\"display:none;\">\n",
+              "        \n",
+              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
+              "       width=\"24px\">\n",
+              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
+              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
+              "  </svg>\n",
+              "      </button>\n",
+              "      \n",
+              "  <style>\n",
+              "    .colab-df-container {\n",
+              "      display:flex;\n",
+              "      flex-wrap:wrap;\n",
+              "      gap: 12px;\n",
+              "    }\n",
+              "\n",
+              "    .colab-df-convert {\n",
+              "      background-color: #E8F0FE;\n",
+              "      border: none;\n",
+              "      border-radius: 50%;\n",
+              "      cursor: pointer;\n",
+              "      display: none;\n",
+              "      fill: #1967D2;\n",
+              "      height: 32px;\n",
+              "      padding: 0 0 0 0;\n",
+              "      width: 32px;\n",
+              "    }\n",
+              "\n",
+              "    .colab-df-convert:hover {\n",
+              "      background-color: #E2EBFA;\n",
+              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
+              "      fill: #174EA6;\n",
+              "    }\n",
+              "\n",
+              "    [theme=dark] .colab-df-convert {\n",
+              "      background-color: #3B4455;\n",
+              "      fill: #D2E3FC;\n",
+              "    }\n",
+              "\n",
+              "    [theme=dark] .colab-df-convert:hover {\n",
+              "      background-color: #434B5C;\n",
+              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
+              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
+              "      fill: #FFFFFF;\n",
+              "    }\n",
+              "  </style>\n",
+              "\n",
+              "      <script>\n",
+              "        const buttonEl =\n",
+              "          document.querySelector('#df-32f1a583-4613-4255-bca3-f005286dd3e1 button.colab-df-convert');\n",
+              "        buttonEl.style.display =\n",
+              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
+              "\n",
+              "        async function convertToInteractive(key) {\n",
+              "          const element = document.querySelector('#df-32f1a583-4613-4255-bca3-f005286dd3e1');\n",
+              "          const dataTable =\n",
+              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
+              "                                                     [key], {});\n",
+              "          if (!dataTable) return;\n",
+              "\n",
+              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
+              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
+              "            + ' to learn more about interactive tables.';\n",
+              "          element.innerHTML = '';\n",
+              "          dataTable['output_type'] = 'display_data';\n",
+              "          await google.colab.output.renderOutput(dataTable, element);\n",
+              "          const docLink = document.createElement('div');\n",
+              "          docLink.innerHTML = docLinkHtml;\n",
+              "          element.appendChild(docLink);\n",
+              "        }\n",
+              "      </script>\n",
+              "    </div>\n",
+              "  </div>\n",
+              "  "
+            ]
+          },
+          "metadata": {},
+          "execution_count": 5
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "amazon_test.tail()"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 206
+        },
+        "id": "UbGZScrEof99",
+        "outputId": "ecd44fe7-805c-403d-bf9b-242995a2ab1f"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "execute_result",
+          "data": {
+            "text/plain": [
+              "         id                                           sentence  label\n",
+              "1995  63354  Big sale this week. All sort of little gadets ...      0\n",
+              "1996  45423  The new owner and management are great. I didn...      0\n",
+              "1997  12024  Came here to check out their Patio Furniture. ...      0\n",
+              "1998  89218  I brought in a flash drive with a 3-page docum...      0\n",
+              "1999  45672  Super helpful. Taught me exactly how to gel st...      0"
+            ],
+            "text/html": [
+              "\n",
+              "  <div id=\"df-c400a5e5-f4dd-4658-b3ec-3e859ac33acf\">\n",
+              "    <div class=\"colab-df-container\">\n",
+              "      <div>\n",
+              "<style scoped>\n",
+              "    .dataframe tbody tr th:only-of-type {\n",
+              "        vertical-align: middle;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe tbody tr th {\n",
+              "        vertical-align: top;\n",
+              "    }\n",
+              "\n",
+              "    .dataframe thead th {\n",
+              "        text-align: right;\n",
+              "    }\n",
+              "</style>\n",
+              "<table border=\"1\" class=\"dataframe\">\n",
+              "  <thead>\n",
+              "    <tr style=\"text-align: right;\">\n",
+              "      <th></th>\n",
+              "      <th>id</th>\n",
+              "      <th>sentence</th>\n",
+              "      <th>label</th>\n",
+              "    </tr>\n",
+              "  </thead>\n",
+              "  <tbody>\n",
+              "    <tr>\n",
+              "      <th>1995</th>\n",
+              "      <td>63354</td>\n",
+              "      <td>Big sale this week. All sort of little gadets ...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>1996</th>\n",
+              "      <td>45423</td>\n",
+              "      <td>The new owner and management are great. I didn...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>1997</th>\n",
+              "      <td>12024</td>\n",
+              "      <td>Came here to check out their Patio Furniture. ...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>1998</th>\n",
+              "      <td>89218</td>\n",
+              "      <td>I brought in a flash drive with a 3-page docum...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "    <tr>\n",
+              "      <th>1999</th>\n",
+              "      <td>45672</td>\n",
+              "      <td>Super helpful. Taught me exactly how to gel st...</td>\n",
+              "      <td>0</td>\n",
+              "    </tr>\n",
+              "  </tbody>\n",
+              "</table>\n",
+              "</div>\n",
+              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-c400a5e5-f4dd-4658-b3ec-3e859ac33acf')\"\n",
+              "              title=\"Convert this dataframe to an interactive table.\"\n",
+              "              style=\"display:none;\">\n",
+              "        \n",
+              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
+              "       width=\"24px\">\n",
+              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
+              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
+              "  </svg>\n",
+              "      </button>\n",
+              "      \n",
+              "  <style>\n",
+              "    .colab-df-container {\n",
+              "      display:flex;\n",
+              "      flex-wrap:wrap;\n",
+              "      gap: 12px;\n",
+              "    }\n",
+              "\n",
+              "    .colab-df-convert {\n",
+              "      background-color: #E8F0FE;\n",
+              "      border: none;\n",
+              "      border-radius: 50%;\n",
+              "      cursor: pointer;\n",
+              "      display: none;\n",
+              "      fill: #1967D2;\n",
+              "      height: 32px;\n",
+              "      padding: 0 0 0 0;\n",
+              "      width: 32px;\n",
+              "    }\n",
+              "\n",
+              "    .colab-df-convert:hover {\n",
+              "      background-color: #E2EBFA;\n",
+              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
+              "      fill: #174EA6;\n",
+              "    }\n",
+              "\n",
+              "    [theme=dark] .colab-df-convert {\n",
+              "      background-color: #3B4455;\n",
+              "      fill: #D2E3FC;\n",
+              "    }\n",
+              "\n",
+              "    [theme=dark] .colab-df-convert:hover {\n",
+              "      background-color: #434B5C;\n",
+              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
+              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
+              "      fill: #FFFFFF;\n",
+              "    }\n",
+              "  </style>\n",
+              "\n",
+              "      <script>\n",
+              "        const buttonEl =\n",
+              "          document.querySelector('#df-c400a5e5-f4dd-4658-b3ec-3e859ac33acf button.colab-df-convert');\n",
+              "        buttonEl.style.display =\n",
+              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
+              "\n",
+              "        async function convertToInteractive(key) {\n",
+              "          const element = document.querySelector('#df-c400a5e5-f4dd-4658-b3ec-3e859ac33acf');\n",
+              "          const dataTable =\n",
+              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
+              "                                                     [key], {});\n",
+              "          if (!dataTable) return;\n",
+              "\n",
+              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
+              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
+              "            + ' to learn more about interactive tables.';\n",
+              "          element.innerHTML = '';\n",
+              "          dataTable['output_type'] = 'display_data';\n",
+              "          await google.colab.output.renderOutput(dataTable, element);\n",
+              "          const docLink = document.createElement('div');\n",
+              "          docLink.innerHTML = docLinkHtml;\n",
+              "          element.appendChild(docLink);\n",
+              "        }\n",
+              "      </script>\n",
+              "    </div>\n",
+              "  </div>\n",
+              "  "
+            ]
+          },
+          "metadata": {},
+          "execution_count": 6
+        }
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "amazon_val.tail()"
+      ],
+      "metadata": {
+        "id": "37a7puk45rdg"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "x_train_full = amazon_train[\"reviewText\"]\n",
+        "y_train_full = amazon_train[\"label\"]\n",
+        "x_val = amazon_val[\"reviewText\"]\n",
+        "y_val = amazon_val[\"label\"]\n",
+        "x_test = amazon_test[\"reviewText\"]\n",
+        "y_test = amazon_test[\"label\"]"
+      ],
+      "metadata": {
+        "id": "vvzKO4Wj4f6c"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "print(f\"Shape x_train: {x_train.shape}\")\n",
+        "print(f\"Shape x_val: {x_val.shape}\")\n",
+        "print(f\"Shape x_test: {x_test.shape}\")\n",
+        "print(f\"Shape y_train: {y_train.shape}\")\n",
+        "print(f\"Shape y_val: {y_val.shape}\")\n",
+        "print(f\"Shape y_test: {y_test.shape}\")"
+      ],
+      "metadata": {
+        "colab": {
+          "base_uri": "https://localhost:8080/"
+        },
+        "id": "GpZ1130U4yLK",
+        "outputId": "03d8afd0-4b78-4793-e18c-73d4fc76d27b"
+      },
+      "execution_count": null,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "name": "stdout",
+          "text": [
+            "Shape X_train: (7200,)\n",
+            "Shape X_valid: (800,)\n",
+            "Shape y_train: (7200,)\n",
+            "Shape y_val: (800,)\n"
+          ]
+        }
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "source": [
+        "## Tokenize inputs"
+      ],
+      "metadata": {
+        "id": "HcyU1oVWakMM"
+      }
+    },
+    {
+      "cell_type": "code",
+      "execution_count": null,
+      "metadata": {
+        "id": "KYbFcFevWm_2",
+        "colab": {
+          "base_uri": "https://localhost:8080/",
+          "height": 113,
+          "referenced_widgets": [
+            "f11fa95cfbf944e1802041360bd570c4",
+            "8ad6f5ce20b54b1998e7564b75110c2c",
+            "915c00323a294011936f9df042b23a3c",
+            "192c6ad0b1f841d59404af12315dcbe0",
+            "3dcc93af7a9942ed90bcf271fcfc9733",
+            "85236db36963455d95b6064d357827a0",
+            "4970a1c799b5404a95dbdaf21381321c",
+            "7c1b3c45dbc64574a8d58857b377c313",
+            "3bc7a7d4fb76484db3c3a8c18560ae59",
+            "ec0b04c10bc4422e89eda112acf5ce94",
+            "9a58e1b736bb4061b0487aabd4787e00",
+            "3b718d0daf134614b0a5acbdb11a36d7",
+            "62d829b69bde49e99c18475c37a05653",
+            "86f761f7538d47d8a1511dcc6f42d0f6",
+            "bab2cbe5504943b495cc0b08575554eb",
+            "4034c1afab21422da245d768659bad36",
+            "5b63cc97705d4f24909deed2cf1d9fe7",
+            "996ebf10aa824f4a80105f6cd3ff6a45",
+            "34859c47432e47568c30a0a4bfa07009",
+            "e28e1c04929a495e8d0d9d04c434a26f",
+            "1b752a32ae58425e96e36d51ac6480f4",
+            "d9a7571d52f540b682250a9a0be70828",
+            "eba887bffb0d4d9b842e5159dfac955f",
+            "1e302ee6f1754b65a33243a32d18949d",
+            "925454fc09394de5a076c4aae58964d5",
+            "3d688a7ceeea4313b942ec99c4b56b16",
+            "0481df8d4eec4acf8ce79bdcef692d65",
+            "ddec1641b47e4f849e8962df58709c9f",
+            "17d6d1d0d479437c9b0658e3e586b68d",
+            "dc61574a6ac14fbf9660c3abf4fd4956",
+            "66cf5b29c3654b8ab290a0faf1e2d1f3",
+            "425ba70930c74f9ab4fe8a709aed6376",
+            "8f2b1dd0f4b44cdba6ee1306be4ed3f5"
+          ]
+        },
+        "outputId": "a1dc2c7e-24e3-4ec4-98ec-2c08e2533a9a"
+      },
+      "outputs": [
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "Downloading:   0%|          | 0.00/226k [00:00<?, ?B/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "f11fa95cfbf944e1802041360bd570c4"
+            }
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "Downloading:   0%|          | 0.00/28.0 [00:00<?, ?B/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "3b718d0daf134614b0a5acbdb11a36d7"
+            }
+          },
+          "metadata": {}
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "Downloading:   0%|          | 0.00/570 [00:00<?, ?B/s]"
+            ],
+            "application/vnd.jupyter.widget-view+json": {
+              "version_major": 2,
+              "version_minor": 0,
+              "model_id": "eba887bffb0d4d9b842e5159dfac955f"
+            }
+          },
+          "metadata": {}
+        }
+      ],
+      "source": [
+        "# Using BERT base uncased tokenizer as per the paper:\n",
+        "bert_tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
+        "\n",
+        "# Use sequence length 320, which achieved best accuracy and F1-score of all sequence lengths tried in the paper:\n",
+        "# https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/4\n",
+        "max_length = 320\n",
+        "\n",
+        "train_encodings = bert_tokenizer(\n",
+        "    list(x_train.values), \n",
+        "    max_length=max_length,\n",
+        "    truncation=True,\n",
+        "    padding='max_length', \n",
+        "    return_tensors='tf'\n",
+        ")\n",
+        "\n",
+        "valid_encodings = bert_tokenizer(\n",
+        "    list(x_val.values), \n",
+        "    max_length=max_length,\n",
+        "    truncation=True,\n",
+        "    padding='max_length', \n",
+        "    return_tensors='tf'\n",
+        ")\n",
+        "\n",
+        "test_encodings = bert_tokenizer(\n",
+        "    list(x_test.values), \n",
+        "    max_length=max_length,\n",
+        "    truncation=True,\n",
+        "    padding='max_length', \n",
+        "    return_tensors='tf'\n",
+        ")"
+      ]
+    },
+    {
+      "cell_type": "markdown",
+      "metadata": {
+        "id": "qQ045x2mH4la"
+      },
+      "source": [
+        "## Model 1: Baseline Amazon product data fine-tuned model\n",
+        "\n",
+        "* All layers unfrozen;\n",
+        "* Same parameters as best Bilal baseline model;\n",
+        "* Trained for 4 epochs."
+      ]
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        "\n",
+        "MODEL_NAME = \"amazon_finetune\"\n",
+        "\n",
+        "\n",
+        "def amazon_finetune():\n",
+        "    \"\"\"Create a BERT model using the model and parameters specified in the Bilal paper:\n",
+        "    https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
+        "\n",
+        "        - model: TFBertForSequenceClassification\n",
+        "        - learning rate: 2e-5\n",
+        "        - epsilon: 1e-8\n",
+        "    \"\"\"\n",
+        "    # Using the TFBertForSequenceClassification as specified in the paper:\n",
+        "    bert_model = TFBertForSequenceClassification.from_pretrained('bert-base-uncased', num_labels=2)\n",
+        "\n",
+        "    # Don't freeze any layers:\n",
+        "    untrainable = []\n",
+        "    trainable = [w.name for w in bert_model.weights]\n",
+        "\n",
+        "    for w in bert_model.weights:\n",
+        "        if w.name in untrainable:\n",
+        "            w._trainable = False\n",
+        "        elif w.name in trainable:\n",
+        "            w._trainable = True\n",
+        "\n",
+        "    # Compile the model:\n",
+        "    bert_model.compile(\n",
+        "        optimizer = tf.keras.optimizers.Adam(learning_rate=2e-5,epsilon=1e-08),\n",
+        "        loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True), \n",
+        "        metrics = [tf.keras.metrics.SparseCategoricalAccuracy(\"accuracy\")]\n",
+        "    )\n",
+        "\n",
+        "    return bert_model\n",
+        "\n",
+        "\n",
+        "model = amazon_finetune()\n",
+        "print(model.summary())\n",
+        "\n",
+        "# Train the model using the specifications from the paper: https://link.springer.com/article/10.1007/s10660-022-09560-w/tables/2\n",
+        "# -- epochs = 4\n",
+        "# -- batch_size = 32\n",
+        "\n",
+        "# Create directory for storing checkpoints after each epoch:\n",
+        "checkpoint_dir = gdrive_save_dir(\"checkpoints\", model_name = MODEL_NAME)\n",
+        "checkpoint_path = checkpoint_dir + \"/cp-{epoch:04d}.ckpt\"\n",
+        "\n",
+        "# Create a callback that saves the model's weights:\n",
+        "cp_callback = tf.keras.callbacks.ModelCheckpoint(\n",
+        "    filepath=checkpoint_path,\n",
+        "    save_weights_only=True,\n",
+        "    verbose=1)\n",
+        "\n",
+        "# Fit the model saving weights every epoch:\n",
+        "history = model.fit(\n",
+        "    [train_encodings.input_ids, train_encodings.token_type_ids, train_encodings.attention_mask], \n",
+        "    y_train.values,\n",
+        "    validation_data=(\n",
+        "        [valid_encodings.input_ids, valid_encodings.token_type_ids, valid_encodings.attention_mask], \n",
+        "        y_val.values\n",
+        "        ),\n",
+        "    batch_size=32, \n",
+        "    epochs=4,\n",
+        "    callbacks=[cp_callback]\n",
+        ")\n",
+        "\n",
+        "# Save the entire model to GDrive:\n",
+        "model_dir = gdrive_save_dir(\"full_model\", model_name = MODEL_NAME)\n",
+        "model.save(model_dir)\n",
+        "\n",
+        "# Save scores on the test set:\n",
+        "test_score = model.evaluate([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask], y_test)\n",
+        "print(\"Test loss:\", test_score[0])\n",
+        "print(\"Test accuracy:\", test_score[1])\n",
+        "score_fp = os.path.join(model_dir, \"test_score.txt\")\n",
+        "with open(score_fp, \"w\") as f:\n",
+        "    f.write(f\"Test loss = {test_score[0]}\\n\")\n",
+        "    f.write(f\"Test accuracy = {test_score[1]}\\n\")\n",
+        "\n",
+        "# Save predictions and classification_report:\n",
+        "predictions = model.predict([test_encodings.input_ids, test_encodings.token_type_ids, test_encodings.attention_mask])\n",
+        "preds_fp = os.path.join(model_dir, \"test_predictions.csv\")\n",
+        "pred_df = pd.DataFrame(predictions.to_tuple()[0], columns=[\"pred_prob_0\", \"pred_prob_1\"])\n",
+        "pred_df[\"yhat\"] = pred_df[[\"pred_prob_0\", \"pred_prob_1\"]].values.argmax(1)\n",
+        "pred_df[\"y\"] = y_test\n",
+        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 1) & (pred_df[\"y\"] == 1), \"tp\", None)\n",
+        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 0) & (pred_df[\"y\"] == 0), \"tn\", pred_df[\"category\"])\n",
+        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 1) & (pred_df[\"y\"] == 0), \"fp\", pred_df[\"category\"])\n",
+        "pred_df[\"category\"] = np.where((pred_df[\"yhat\"] == 0) & (pred_df[\"y\"] == 1), \"fn\", pred_df[\"category\"])\n",
+        "pred_df.to_csv(preds_fp, encoding=\"utf-8\", index=False)\n",
+        "report = classification_report(y_test, pred_df[\"yhat\"])\n",
+        "report_fp = os.path.join(model_dir, \"classification_report.txt\")\n",
+        "with open(report_fp, \"w\") as f:\n",
+        "    for line in report.split(\"\\n\"):\n",
+        "        f.write(f\"{line}\\n\")\n",
+        "print(f\"{MODEL_NAME} - test set results\")\n",
+        "print(report)"
+      ],
+      "metadata": {
+        "id": "nF3jEAt-68jx"
+      },
+      "execution_count": null,
+      "outputs": []
+    },
+    {
+      "cell_type": "code",
+      "source": [
+        ""
+      ],
+      "metadata": {
+        "id": "2j2lhg8ggkY6"
+      },
+      "execution_count": null,
+      "outputs": []
+    }
+  ],
+  "metadata": {
+    "accelerator": "TPU",
+    "colab": {
+      "background_execution": "on",
+      "collapsed_sections": [
+        "XtosRoO6i-Nq",
+        "sL61_-AsltW6"
+      ],
+      "machine_shape": "hm",
+      "name": "Amazon_product_data_transfer_learning.ipynb",
+      "provenance": [],
+      "include_colab_link": true
+    },
+    "gpuClass": "standard",
+    "kernelspec": {
+      "display_name": "Python 3",
+      "name": "python3"
+    },
+    "language_info": {
+      "name": "python"
+    },
+    "widgets": {
+      "application/vnd.jupyter.widget-state+json": {
+        "f11fa95cfbf944e1802041360bd570c4": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_8ad6f5ce20b54b1998e7564b75110c2c",
+              "IPY_MODEL_915c00323a294011936f9df042b23a3c",
+              "IPY_MODEL_192c6ad0b1f841d59404af12315dcbe0"
+            ],
+            "layout": "IPY_MODEL_3dcc93af7a9942ed90bcf271fcfc9733"
+          }
+        },
+        "8ad6f5ce20b54b1998e7564b75110c2c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_85236db36963455d95b6064d357827a0",
+            "placeholder": "​",
+            "style": "IPY_MODEL_4970a1c799b5404a95dbdaf21381321c",
+            "value": "Downloading: 100%"
+          }
+        },
+        "915c00323a294011936f9df042b23a3c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_7c1b3c45dbc64574a8d58857b377c313",
+            "max": 231508,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_3bc7a7d4fb76484db3c3a8c18560ae59",
+            "value": 231508
+          }
+        },
+        "192c6ad0b1f841d59404af12315dcbe0": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_ec0b04c10bc4422e89eda112acf5ce94",
+            "placeholder": "​",
+            "style": "IPY_MODEL_9a58e1b736bb4061b0487aabd4787e00",
+            "value": " 226k/226k [00:00&lt;00:00, 1.86MB/s]"
+          }
+        },
+        "3dcc93af7a9942ed90bcf271fcfc9733": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "85236db36963455d95b6064d357827a0": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "4970a1c799b5404a95dbdaf21381321c": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "7c1b3c45dbc64574a8d58857b377c313": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "3bc7a7d4fb76484db3c3a8c18560ae59": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "ec0b04c10bc4422e89eda112acf5ce94": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "9a58e1b736bb4061b0487aabd4787e00": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "3b718d0daf134614b0a5acbdb11a36d7": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_62d829b69bde49e99c18475c37a05653",
+              "IPY_MODEL_86f761f7538d47d8a1511dcc6f42d0f6",
+              "IPY_MODEL_bab2cbe5504943b495cc0b08575554eb"
+            ],
+            "layout": "IPY_MODEL_4034c1afab21422da245d768659bad36"
+          }
+        },
+        "62d829b69bde49e99c18475c37a05653": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_5b63cc97705d4f24909deed2cf1d9fe7",
+            "placeholder": "​",
+            "style": "IPY_MODEL_996ebf10aa824f4a80105f6cd3ff6a45",
+            "value": "Downloading: 100%"
+          }
+        },
+        "86f761f7538d47d8a1511dcc6f42d0f6": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_34859c47432e47568c30a0a4bfa07009",
+            "max": 28,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_e28e1c04929a495e8d0d9d04c434a26f",
+            "value": 28
+          }
+        },
+        "bab2cbe5504943b495cc0b08575554eb": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_1b752a32ae58425e96e36d51ac6480f4",
+            "placeholder": "​",
+            "style": "IPY_MODEL_d9a7571d52f540b682250a9a0be70828",
+            "value": " 28.0/28.0 [00:00&lt;00:00, 965B/s]"
+          }
+        },
+        "4034c1afab21422da245d768659bad36": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "5b63cc97705d4f24909deed2cf1d9fe7": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "996ebf10aa824f4a80105f6cd3ff6a45": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "34859c47432e47568c30a0a4bfa07009": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "e28e1c04929a495e8d0d9d04c434a26f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "1b752a32ae58425e96e36d51ac6480f4": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "d9a7571d52f540b682250a9a0be70828": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "eba887bffb0d4d9b842e5159dfac955f": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HBoxModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HBoxModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HBoxView",
+            "box_style": "",
+            "children": [
+              "IPY_MODEL_1e302ee6f1754b65a33243a32d18949d",
+              "IPY_MODEL_925454fc09394de5a076c4aae58964d5",
+              "IPY_MODEL_3d688a7ceeea4313b942ec99c4b56b16"
+            ],
+            "layout": "IPY_MODEL_0481df8d4eec4acf8ce79bdcef692d65"
+          }
+        },
+        "1e302ee6f1754b65a33243a32d18949d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_ddec1641b47e4f849e8962df58709c9f",
+            "placeholder": "​",
+            "style": "IPY_MODEL_17d6d1d0d479437c9b0658e3e586b68d",
+            "value": "Downloading: 100%"
+          }
+        },
+        "925454fc09394de5a076c4aae58964d5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "FloatProgressModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "FloatProgressModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "ProgressView",
+            "bar_style": "success",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_dc61574a6ac14fbf9660c3abf4fd4956",
+            "max": 570,
+            "min": 0,
+            "orientation": "horizontal",
+            "style": "IPY_MODEL_66cf5b29c3654b8ab290a0faf1e2d1f3",
+            "value": 570
+          }
+        },
+        "3d688a7ceeea4313b942ec99c4b56b16": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "HTMLModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_dom_classes": [],
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "HTMLModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/controls",
+            "_view_module_version": "1.5.0",
+            "_view_name": "HTMLView",
+            "description": "",
+            "description_tooltip": null,
+            "layout": "IPY_MODEL_425ba70930c74f9ab4fe8a709aed6376",
+            "placeholder": "​",
+            "style": "IPY_MODEL_8f2b1dd0f4b44cdba6ee1306be4ed3f5",
+            "value": " 570/570 [00:00&lt;00:00, 20.8kB/s]"
+          }
+        },
+        "0481df8d4eec4acf8ce79bdcef692d65": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "ddec1641b47e4f849e8962df58709c9f": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "17d6d1d0d479437c9b0658e3e586b68d": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        },
+        "dc61574a6ac14fbf9660c3abf4fd4956": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "66cf5b29c3654b8ab290a0faf1e2d1f3": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "ProgressStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "ProgressStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "bar_color": null,
+            "description_width": ""
+          }
+        },
+        "425ba70930c74f9ab4fe8a709aed6376": {
+          "model_module": "@jupyter-widgets/base",
+          "model_name": "LayoutModel",
+          "model_module_version": "1.2.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/base",
+            "_model_module_version": "1.2.0",
+            "_model_name": "LayoutModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "LayoutView",
+            "align_content": null,
+            "align_items": null,
+            "align_self": null,
+            "border": null,
+            "bottom": null,
+            "display": null,
+            "flex": null,
+            "flex_flow": null,
+            "grid_area": null,
+            "grid_auto_columns": null,
+            "grid_auto_flow": null,
+            "grid_auto_rows": null,
+            "grid_column": null,
+            "grid_gap": null,
+            "grid_row": null,
+            "grid_template_areas": null,
+            "grid_template_columns": null,
+            "grid_template_rows": null,
+            "height": null,
+            "justify_content": null,
+            "justify_items": null,
+            "left": null,
+            "margin": null,
+            "max_height": null,
+            "max_width": null,
+            "min_height": null,
+            "min_width": null,
+            "object_fit": null,
+            "object_position": null,
+            "order": null,
+            "overflow": null,
+            "overflow_x": null,
+            "overflow_y": null,
+            "padding": null,
+            "right": null,
+            "top": null,
+            "visibility": null,
+            "width": null
+          }
+        },
+        "8f2b1dd0f4b44cdba6ee1306be4ed3f5": {
+          "model_module": "@jupyter-widgets/controls",
+          "model_name": "DescriptionStyleModel",
+          "model_module_version": "1.5.0",
+          "state": {
+            "_model_module": "@jupyter-widgets/controls",
+            "_model_module_version": "1.5.0",
+            "_model_name": "DescriptionStyleModel",
+            "_view_count": null,
+            "_view_module": "@jupyter-widgets/base",
+            "_view_module_version": "1.2.0",
+            "_view_name": "StyleView",
+            "description_width": ""
+          }
+        }
+      }
+    }
+  },
+  "nbformat": 4,
+  "nbformat_minor": 0
+}
\ No newline at end of file