NLP/DeBERTa Fine Tuning-for Amazon Review Dataset Pytorch.ipynb

{
 "cells": [
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "848860a3",
   "metadata": {},
   "source": [
    "<a href=\"https://www.youtube.com/watch?v=-rqmj_tfQLo&list=PLxqBkZuBynVQEvXfJpq3smfuKq3AiNW-N&index=34&ab_channel=Rohan-Paul-AI\"><h1 style=\"font-size:250%; font-family:cursive; color:#ff6666;\"><b>Link to my YouTube Video - Debarta fine tuning for Amazon Review Dataset | NLP</b></h1></a>\n",
    "\n",
    "[![IMAGE ALT TEXT](https://imgur.com/WbaxGoM.png)](https://www.youtube.com/watch?v=-rqmj_tfQLo&list=PLxqBkZuBynVQEvXfJpq3smfuKq3AiNW-N&index=34&ab_channel=Rohan-Paul-AI)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "ce5c8e9f",
   "metadata": {
    "papermill": {
     "duration": 0.029399,
     "end_time": "2022-06-01T14:00:08.104814",
     "exception": false,
     "start_time": "2022-06-01T14:00:08.075415",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "### [Data Source](https://github.com/rohan-paul/YT_Data_Sources/blob/main/Amazon-Review-Text-NLP/amazon-review/cleaned_reviews.csv)\n",
    "\n",
    "-----------------------\n",
    "\n",
    "### Key parts of this Fine Tuning Strategies\n",
    "\n",
    "1. Mean Pooling\n",
    "2. Gradient Accumulation\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f848df56",
   "metadata": {},
   "outputs": [],
   "source": [
    "# !pip install --upgrade wandb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "1fb7bd59",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:00:23.286100Z",
     "iopub.status.busy": "2022-06-01T14:00:23.285437Z",
     "iopub.status.idle": "2022-06-01T14:00:30.888727Z",
     "shell.execute_reply": "2022-06-01T14:00:30.887936Z"
    },
    "papermill": {
     "duration": 7.642363,
     "end_time": "2022-06-01T14:00:30.891351",
     "exception": false,
     "start_time": "2022-06-01T14:00:23.248988",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import gc\n",
    "import copy\n",
    "import time\n",
    "import random\n",
    "import string\n",
    "import joblib\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.optim as optim\n",
    "from torch.optim import lr_scheduler\n",
    "from torch.utils.data import Dataset, DataLoader\n",
    "\n",
    "# Utils\n",
    "from tqdm import tqdm\n",
    "from collections import defaultdict\n",
    "\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.model_selection import GroupKFold, KFold\n",
    "\n",
    "from transformers import AutoTokenizer, AutoModel, AutoConfig, AdamW\n",
    "from transformers import DataCollatorWithPadding\n",
    "\n",
    "from colorama import Fore, Back, Style\n",
    "\n",
    "b_ = Fore.BLUE\n",
    "y_ = Fore.YELLOW\n",
    "sr_ = Style.RESET_ALL\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "\n",
    "os.environ['CUDA_LAUNCH_BLOCKING'] = \"1\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e81d424b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \n",
      "Get your W&B access token from here: https://wandb.ai/authorize\n"
     ]
    }
   ],
   "source": [
    "import wandb\n",
    "\n",
    "try:\n",
    "    from kaggle_secrets import UserSecretsClient\n",
    "    user_secrets = UserSecretsClient()\n",
    "    api_key = user_secrets.get_secret(\"wandb_api\")\n",
    "    wandb.login(key=api_key)\n",
    "    anony = None\n",
    "except:\n",
    "    anony = \"must\"\n",
    "    print('If you want to use your W&B account, go to Add-ons -> Secrets and provide your W&B access token. Use the Label name as wandb_api. \\nGet your W&B access token from here: https://wandb.ai/authorize')"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "c9ed1950",
   "metadata": {
    "papermill": {
     "duration": 0.037298,
     "end_time": "2022-06-01T14:00:33.132796",
     "exception": false,
     "start_time": "2022-06-01T14:00:33.095498",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Training Configuration ⚙️ </h3>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "3201fd82",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:00:33.207330Z",
     "iopub.status.busy": "2022-06-01T14:00:33.207080Z",
     "iopub.status.idle": "2022-06-01T14:00:33.213413Z",
     "shell.execute_reply": "2022-06-01T14:00:33.212504Z"
    },
    "papermill": {
     "duration": 0.045724,
     "end_time": "2022-06-01T14:00:33.215144",
     "exception": false,
     "start_time": "2022-06-01T14:00:33.169420",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "qejd0jcovrw2\n"
     ]
    }
   ],
   "source": [
    "def id_generator(size=12, chars=string.ascii_lowercase + string.digits):\n",
    "    return ''.join(random.SystemRandom().choice(chars) for _ in range(size))\n",
    "\n",
    "HASH_NAME = id_generator(size=12)\n",
    "print(HASH_NAME)"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "5fbe8779",
   "metadata": {},
   "source": [
    "The function `id_generator` is designed to create a random string with a specified length (default is 12 characters) from a set of provided characters. The whole purpose is to generate a unique, random identifier.\n",
    "\n",
    "`string.ascii_lowercase + string.digits` => all the lowercase ASCII characters (a-z) and digits (0-9)\n",
    "\n",
    "`random.SystemRandom()` is a method that uses sources provided by the operating system (like /dev/urandom in Unix) to generate random numbers. It's a way of generating random numbers that's suitable for cryptographic use, as it's more secure than the standard random number generator in Python. choice(chars) then selects a single character randomly from the set chars.\n",
    "\n",
    "`'.join(.... for _ in range(size))` The for loop inside the join function runs for size number of times (which is 12 by default) and in each iteration, it calls random.SystemRandom().choice(chars) to select a character. These characters are then joined together into a single string using the join function, which connects them without any additional characters in between. The result is a randomly generated string of a specified length."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "49d339cd",
   "metadata": {},
   "outputs": [],
   "source": [
    "ROOT_PATH = '../input/amazon-review/cleaned_reviews.csv' # Local Machine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0e6f5763",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:00:33.453101Z",
     "iopub.status.busy": "2022-06-01T14:00:33.452894Z",
     "iopub.status.idle": "2022-06-01T14:00:38.828476Z",
     "shell.execute_reply": "2022-06-01T14:00:38.827637Z"
    },
    "papermill": {
     "duration": 5.414865,
     "end_time": "2022-06-01T14:00:38.830316",
     "exception": false,
     "start_time": "2022-06-01T14:00:33.415451",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n",
      "Special tokens have been added in the vocabulary, make sure the associated word embeddings are fine-tuned or trained.\n"
     ]
    }
   ],
   "source": [
    "CONFIG = {\"seed\": 2022,\n",
    "          \"epochs\": 3,\n",
    "          \"model_name\": \"microsoft/deberta-v3-base\",\n",
    "          \"train_batch_size\": 8,\n",
    "          \"valid_batch_size\": 16,\n",
    "          \"max_length\": 512,\n",
    "          \"learning_rate\": 1e-5,\n",
    "          \"scheduler\": 'CosineAnnealingLR',\n",
    "          \"min_lr\": 1e-6,\n",
    "          \"T_max\": 500,\n",
    "          \"weight_decay\": 1e-6,\n",
    "          \"n_fold\": 3,\n",
    "          \"n_accumulate\": 1,\n",
    "          \"num_classes\": 3,\n",
    "          \"device\": torch.device(\"cuda:0\" if torch.cuda.is_available() else \"cpu\"),\n",
    "          \"hash_name\": HASH_NAME,\n",
    "          \"competition\": \"amazon-reviews-dataset\",\n",
    "          \"_wandb_kernel\": \"react\",\n",
    "          }\n",
    "\n",
    "CONFIG[\"tokenizer\"] = AutoTokenizer.from_pretrained(CONFIG['model_name'])\n",
    "CONFIG['group'] = f'{HASH_NAME}-Baseline'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b4f208e2",
   "metadata": {},
   "outputs": [],
   "source": [
    "def set_seeds(config):\n",
    "    '''Sets the seed of the entire program so results are the same every time we run.\n",
    "    This is for REPRODUCIBILITY.'''\n",
    "    seed = config['seed']\n",
    "    np.random.seed(seed)\n",
    "    torch.manual_seed(seed)\n",
    "\n",
    "    if torch.cuda.is_available():\n",
    "        torch.cuda.manual_seed(seed)\n",
    "        torch.backends.cudnn.deterministic = True\n",
    "        torch.backends.cudnn.benchmark = False\n",
    "        \n",
    "    os.environ['PYTHONHASHSEED'] = str(seed)\n",
    "\n",
    "# Usage\n",
    "set_seeds(CONFIG)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7fa2fcc0",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:00:39.239856Z",
     "iopub.status.busy": "2022-06-01T14:00:39.239195Z",
     "iopub.status.idle": "2022-06-01T14:01:04.468382Z",
     "shell.execute_reply": "2022-06-01T14:01:04.467736Z"
    },
    "papermill": {
     "duration": 25.269923,
     "end_time": "2022-06-01T14:01:04.470131",
     "exception": false,
     "start_time": "2022-06-01T14:00:39.200208",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sentiments</th>\n",
       "      <th>cleaned_review</th>\n",
       "      <th>cleaned_review_length</th>\n",
       "      <th>review_score</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>positive</td>\n",
       "      <td>i wish would have gotten one earlier love it a...</td>\n",
       "      <td>19</td>\n",
       "      <td>5.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>neutral</td>\n",
       "      <td>i ve learned this lesson again open the packag...</td>\n",
       "      <td>88</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>neutral</td>\n",
       "      <td>it is so slow and lags find better option</td>\n",
       "      <td>9</td>\n",
       "      <td>2.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>neutral</td>\n",
       "      <td>roller ball stopped working within months of m...</td>\n",
       "      <td>12</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>neutral</td>\n",
       "      <td>i like the color and size but it few days out ...</td>\n",
       "      <td>21</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  sentiments                                     cleaned_review   \n",
       "0   positive  i wish would have gotten one earlier love it a...  \\\n",
       "1    neutral  i ve learned this lesson again open the packag...   \n",
       "2    neutral          it is so slow and lags find better option   \n",
       "3    neutral  roller ball stopped working within months of m...   \n",
       "4    neutral  i like the color and size but it few days out ...   \n",
       "\n",
       "   cleaned_review_length  review_score  \n",
       "0                     19           5.0  \n",
       "1                     88           1.0  \n",
       "2                      9           2.0  \n",
       "3                     12           1.0  \n",
       "4                     21           1.0  "
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df = pd.read_csv(ROOT_PATH)\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "0b3a9baf",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df.shape  (17340, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "3"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "print('df.shape ', df.shape)\n",
    "nan_count = df['cleaned_review'].isna().sum()\n",
    "\n",
    "nan_count\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "ff5cfde6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "df.shape  (17337, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# df['cleaned_review'] = df['cleaned_review'].fillna(' ')\n",
    "df.dropna(inplace=True)\n",
    "df.reset_index(drop=True, inplace=True)\n",
    "# Without resetting, you will get error something like - KeyError: '[2085] not in index' \n",
    "# which indicates that the index of the DataFrame df does not include the value 2085. \n",
    "# This might be happening because the DataFrame df has fewer rows than that or rows have been dropped and the index is not reset.\n",
    "\n",
    "print('df.shape ', df.shape)\n",
    "\n",
    "nan_count_after = df['cleaned_review'].isna().sum()\n",
    "nan_count_after"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "54db71dc",
   "metadata": {},
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> GroupKFold </h3>\n",
    "</div>\n",
    "\n",
    "## Whats the speciality of GroupKFold and where is it used\n",
    "\n",
    "GroupKFold is particularly useful when dealing with data that have a group structure, and data leakage between the train and test sets may occur if the group structure is not taken into account.\n",
    "\n",
    "### Data leakage is when information from outside the training dataset is used to train the model. This can easily happen when there's a group structure in your data, and some data from one group ends up in the training set, and some in the test set. The model might then learn about specific groups, rather than generalizing well to unseen data.\n",
    "\n",
    "For example:\n",
    "\n",
    "**Patient data in healthcare:** Here, multiple data points might be collected from the same patient. These data points could share certain characteristics that can influence a model. If we have data from the same patient in both the training and testing set, our model might perform overly well because it has indirectly seen the test data during training.\n",
    "\n",
    "**Sentences or documents in NLP:** When working with text data, it is common to split documents into sentences or parts, each part as an individual data point. Here, if we have some sentences from the same document in the training set and some in the test set, our model might perform better than expected because sentences in the same document are often related.\n",
    "\n",
    "**Time-series data:** For time-series data, the same subject (like a place or a person) can have multiple measurements at different points in time. If data from the same subject ends up in both training and testing sets, it can lead to overly optimistic performance estimates.\n",
    "\n",
    "In these cases, using GroupKFold can help get a more accurate measure of a model's performance by making sure all data points from one group end up in either the training set or the test set, but not both. The model is then forced to learn more general patterns that apply to unseen groups, rather than specific patterns within a group.\n",
    "\n",
    "### Now, I want to add a new column named \"kfold\" to your dataframe, where each row gets the fold number it belongs to."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4195d9a9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Assign group ids to unique reviews\n",
    "# We create a numerical group identifier, as this dataframe does not have a unique identifier for each row\n",
    "# This identifier will be created by assigning unique ids to unique reviews.\n",
    "df['group'] = df['cleaned_review'].factorize()[0]\n",
    "\n",
    "\n",
    "# Initialize GroupKFold\n",
    "gkf = GroupKFold(n_splits=CONFIG['n_fold'])\n",
    "\n",
    "# Apply group k-fold\n",
    "for fold, (_, val_index) in enumerate(gkf.split(X=df, groups=df['group'])):\n",
    "    df.loc[val_index, \"kfold\"] = int(fold)\n",
    "    \n",
    "df[\"kfold\"] = df[\"kfold\"].astype(int)\n",
    "df.head()"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "6b33f103",
   "metadata": {},
   "source": [
    "\n",
    "## `pandas.factorize()`\n",
    "\n",
    "pandas.factorize() function is used to identify distinct values and assign them a numerical identifier. This can be useful in scenarios where we want to convert categorical data into a numerical format\n",
    "\n",
    "It returns two outputs: a label array and an array with unique values. In simple terms, it assigns a unique integer to each unique value in the array.\n",
    "\n",
    "```py\n",
    "\n",
    "s = pd.Series(['cat', 'dog', 'cat', 'dog', 'bird', 'bird', 'cat'])\n",
    "\n",
    "labels, uniques = pd.factorize(s)\n",
    "\n",
    "print(\"Labels:\", labels) # Labels: [0 1 0 1 2 2 0]\n",
    "print(\"Uniques:\", uniques) # Uniques: ['cat', 'dog', 'bird']\n",
    "\n",
    "```\n",
    "\n",
    "factorize() deals with NaN (missing values) as a distinct category. So if your series had any NaN values, they would be factorized into a unique number as well. If you don't want this behavior, you would have to handle missing values prior to calling factorize().\n",
    "\n",
    "In our case above, `df['group'] = df['cleaned_review'].factorize()[0]`\n",
    "\n",
    "- This is creating a new column called 'group' in the dataframe df, and assigning to it the numerical labels returned by factorize().\n",
    "\n",
    "- So it creates a group identifier for each unique 'cleaned_review'. These group identifiers can then be used in the GroupKFold process to ensure that all entries from a particular unique review are either in the training set or the validation set, but not both.\n",
    "\n",
    "===================================================\n",
    "\n",
    "\n",
    "\n",
    "## Explanation of `for fold, (_, val_index) in enumerate(gkf.split(X=df, groups=df['group'])):`\n",
    "\n",
    "gkf.split() returns a generator that produces indices that can be used to generate dataset splits.\n",
    "\n",
    "The split() function takes three arguments, but in this case, we only provide two: X, which is the dataset to split, and groups, which is an array-like object that defines the groups within the data. \n",
    "\n",
    "**`gkf.split(X=df, groups=df['group'])` will return two lists of indices for each split**: the indices of the rows in the training set (which we're not using, hence the underscore _), and the indices of the rows in the validation set (val_index).\n",
    "\n",
    "`enumerate()` is applied to the results from `gkf.split()`. It returns tuples where the first element is the count (starting from 0), and the second element is the value from the iterable. Here, the count corresponds to the fold number and is stored in fold.\n",
    "\n",
    "===================================================\n",
    "\n",
    "\n",
    "\n",
    "## Explanation of `df.loc[val_index, \"kfold\"] = int(fold)`\n",
    "\n",
    "df.loc[val_index, \"kfold\"] selects rows and the \"kfold\" column in the DataFrame df. Here, val_index are the row indices for the validation set for the current fold, returned by gkf.split().\n",
    "\n",
    "int(fold) converts the fold number (originally a float) to an integer. This is then assigned to the \"kfold\" column of the DataFrame at the positions specified by val_index.\n",
    "\n",
    "So, essentially, this line is labeling the rows in the DataFrame that belong to the current fold's validation set with the fold number.\n",
    "\n",
    "The significance of these lines is that they're assigning each row in the DataFrame to a fold number (0 through n_splits-1). These fold numbers can be used for K-Fold cross-validation, where in each iteration (or \"fold\"), a different subset of the data is held out for validation while the model is trained on the rest of the data. The grouping ensures that all samples with the same group value end up in either the training or the validation set, but not in both, preventing potential data leakage."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f53c378",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:04.792249Z",
     "iopub.status.busy": "2022-06-01T14:01:04.792042Z",
     "iopub.status.idle": "2022-06-01T14:01:04.812272Z",
     "shell.execute_reply": "2022-06-01T14:01:04.811470Z"
    },
    "papermill": {
     "duration": 0.061909,
     "end_time": "2022-06-01T14:01:04.814050",
     "exception": false,
     "start_time": "2022-06-01T14:01:04.752141",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "df.groupby('kfold')['sentiments'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f1a3964",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:04.899447Z",
     "iopub.status.busy": "2022-06-01T14:01:04.899240Z",
     "iopub.status.idle": "2022-06-01T14:01:04.916611Z",
     "shell.execute_reply": "2022-06-01T14:01:04.915891Z"
    },
    "papermill": {
     "duration": 0.060795,
     "end_time": "2022-06-01T14:01:04.918551",
     "exception": false,
     "start_time": "2022-06-01T14:01:04.857756",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# Convert from categorical values ('positive', 'negative', 'neutral') to numerical values\n",
    "encoder = LabelEncoder()\n",
    "df['sentiments'] = encoder.fit_transform(df['sentiments'])\n",
    "\n",
    "\"\"\" fit_transform() does two things: it first 'fits' the encoder by learning the mapping between classes and integer labels (using the fit method), and then it 'transforms' the input data into its encoded form (using the transform method).\n",
    "\n",
    "So, fit_transform(df['sentiments']) is learning the mapping from 'positive', 'negative', and 'neutral' to integer labels, and then immediately applying this mapping to df['sentiments'] to produce a new numpy array of encoded labels.\n",
    "\n",
    "So this above line, replaces the original 'sentiments' column in the dataframe with the newly encoded numpy array. Now, instead of 'positive', 'negative', and 'neutral', this column contains integer labels that represent these classes. \"\"\"\n",
    "\n",
    "with open(\"le.pkl\", \"wb\") as fp:\n",
    "    joblib.dump(encoder, fp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ad36054",
   "metadata": {},
   "outputs": [],
   "source": [
    "class TextDataset(Dataset):\n",
    "    def __init__(self, df, tokenizer, max_length):\n",
    "        self.df = df\n",
    "        self.max_len = max_length\n",
    "        self.tokenizer = tokenizer\n",
    "        self.cleaned_review = df['cleaned_review'].values\n",
    "        self.targets = df['sentiments'].values\n",
    "        \n",
    "    def __len__(self):\n",
    "        return len(self.df)\n",
    "    \n",
    "    \"\"\" The __getitem__ method essentially retrieves the data sample (inputs and target) at a specific index and processes it into a format that can be used by the BERT model.\n",
    "    This method is particularly important because PyTorch's DataLoader class uses this method to create the batches during training and evaluation. When you pass your Dataset to a DataLoader and start iterating over it, the DataLoader will call the __getitem__ method to generate the necessary inputs for your model.\"\"\"\n",
    "    def __getitem__(self, index):\n",
    "        cleaned_review = self.cleaned_review[index]\n",
    "        text = self.tokenizer.sep_token + \" \" + cleaned_review\n",
    "        # Above line appends the [SEP] token to the start of the review. This is a requirement of the BERT model, which uses the [SEP] token to differentiate between different sentences.\n",
    "        \n",
    "        # The preprocessed text is then tokenized using the encode_plus method of the tokenizer. The tokenized inputs are truncated to a maximum length of self.max_len and the special tokens [CLS] and [SEP] are added to the sequence.\n",
    "        inputs = self.tokenizer.encode_plus(\n",
    "                        text,\n",
    "                        truncation=True,\n",
    "                        add_special_tokens=True,\n",
    "                        max_length=self.max_len\n",
    "                    )\n",
    "        \n",
    "        return {\n",
    "            'input_ids': inputs['input_ids'],\n",
    "            'attention_mask': inputs['attention_mask'],\n",
    "            'target': self.targets[index]\n",
    "        }"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5ddf6853",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:05.180324Z",
     "iopub.status.busy": "2022-06-01T14:01:05.180107Z",
     "iopub.status.idle": "2022-06-01T14:01:05.183679Z",
     "shell.execute_reply": "2022-06-01T14:01:05.182914Z"
    },
    "papermill": {
     "duration": 0.045869,
     "end_time": "2022-06-01T14:01:05.185603",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.139734",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "collate_fn = DataCollatorWithPadding(tokenizer=CONFIG['tokenizer'])"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "40c6008c",
   "metadata": {
    "papermill": {
     "duration": 0.039617,
     "end_time": "2022-06-01T14:01:05.264786",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.225169",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Mean Pooling </h3>\n",
    "</div>"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "4d01af1d",
   "metadata": {},
   "source": [
    "## Theory behind `MeanPooling(nn.Module)` and its calculation in the context of Transformer NLP project\n",
    "\n",
    "The process of converting a sequence of embeddings into a sentence embedding is called “pooling”. Intuitively, this entails compressing the granular token-level representations into a single fixed-length representation that is supposed to reflect the meaning of the entire sequence.\n",
    "\n",
    "Simply put, The embeddings go through a pooling layer to get a single fixed-length embedding for all the text. For example, mean pooling averages the embeddings generated by the model.\n",
    "\n",
    "![](2023-05-27-16-34-51.png)\n",
    "\n",
    "The MeanPooling class applies mean pooling to the hidden states of a transformer model. **This operation is used to create a single vector representation for an entire input sequence**, which can be used for sequence classification tasks, such as sentiment analysis or spam detection.\n",
    "\n",
    "## So, after applying MeanPooling, each sentence in the batch is represented by a single vector that is the average of the embeddings of its actual tokens. This can be fed into a classifier to predict, for example, the sentiment of the sentence."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3cfc39d2",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:05.344642Z",
     "iopub.status.busy": "2022-06-01T14:01:05.344419Z",
     "iopub.status.idle": "2022-06-01T14:01:05.349769Z",
     "shell.execute_reply": "2022-06-01T14:01:05.349080Z"
    },
    "papermill": {
     "duration": 0.047496,
     "end_time": "2022-06-01T14:01:05.351514",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.304018",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class MeanPooling(nn.Module):\n",
    "    \"\"\" The MeanPooling class inherits from the nn.Module class which is the base class for all neural network modules in PyTorch. \"\"\"\n",
    "    def __init__(self):\n",
    "        super(MeanPooling, self).__init__()\n",
    "        # In above line __init__() is called to initialize the nn.Module parent class.\n",
    "\n",
    "    def forward(self, last_hidden_state, attention_mask):\n",
    "\n",
    "        # attention_mask => (batch_size, sequence_length)\n",
    "        # last_hidden_state represents the output of the transformer model, which is a 3D tensor of shape (batch_size, sequence_length, hidden_size).\n",
    "\n",
    "        # First, the attention_mask is expanded to match the size of the last_hidden_state:\n",
    "        input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float() # => (batch_size, sequence_length, hidden_size).\n",
    "        # The resulting tensor is of shape (batch_size, sequence_length, hidden_size).\n",
    "        #  where each [PAD] token is represented by a vector of zeros, and all other tokens are represented by vectors of ones.\n",
    "\n",
    "        # Then, the last_hidden_state is multiplied by the expanded mask to zero out the embeddings of the [PAD] tokens:\n",
    "        sum_embeddings = torch.sum(last_hidden_state * input_mask_expanded, 1)\n",
    "        # This line computes the sum of the input_mask_expanded along the sequence_length dimension.\n",
    "        # This sum represents the number of actual (non-padding) tokens in each sequence of the batch.\n",
    "\n",
    "        # The sum of the mask values is then computed for each sentence:\n",
    "        # Result: A 2D tensor of shape (batch_size, hidden_size), where each value represents the number of actual tokens (excluding padding tokens) in the corresponding sentence.\n",
    "        sum_mask = input_mask_expanded.sum(1) #=> (batch_size, hidden_size)\n",
    "\n",
    "        \"\"\" In above line, the sum function with argument 1 is called on input_mask_expanded to compute the sum along the sequence_length dimension. Essentially, this operation is adding up all the 1s for each sequence in the batch, which gives us the number of actual tokens (i.e., non-padding tokens) in each sequence.\n",
    "        So, sum_mask is a 2D tensor of shape (batch_size, hidden_size), where each value represents the number of actual tokens in the corresponding sequence.\n",
    "\n",
    "        This is a crucial step in calculating the mean embeddings for each sequence. By summing the mask values, we essentially count the number of valid (non-padding) tokens in each sequence. This count is later used as the denominator when calculating the mean (i.e., sum of token embeddings / number of tokens).\n",
    "\n",
    "        By only considering non-padding tokens, we ensure the mean embeddings accurately represent the sequence, rather than being skewed by padding tokens that carry no meaningful information.\n",
    "        \"\"\"\n",
    "\n",
    "        # a lower limit is set on the sum_mask values to avoid division by zero:\n",
    "        # Result: The same tensor as sum_mask, but any value that was originally zero is now 1e-9.\n",
    "        sum_mask = torch.clamp(sum_mask, min=1e-9)\n",
    "\n",
    "        #Finally, the mean of the embeddings is computed by dividing the sum of the embeddings by the number of actual tokens:\n",
    "        mean_embeddings = sum_embeddings / sum_mask\n",
    "\n",
    "        # Result: A 2D tensor of shape (batch_size, hidden_size), representing the sentence-level embeddings computed as the mean of the token-level embeddings (ignoring padding tokens).\n",
    "        return mean_embeddings"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "3f6f610a",
   "metadata": {},
   "source": [
    "## `input_mask_expanded = attention_mask.unsqueeze(-1).expand(last_hidden_state.size()).float()`\n",
    "\n",
    "\n",
    "last_hidden_state represents the output of the transformer model, which is a 3D tensor of shape **(batch_size, sequence_length, hidden_size)**. This tensor contains the embeddings of all tokens in all sequences of the batch.\n",
    "\n",
    "The attention_mask is a tensor that represents the attention mask for the input sequence. It is a 2-dimensional tensor with a shape of (batch_size, sequence_length). The unsqueeze function is used to add an extra dimension to the tensor at the specified position (-1). This results in a tensor with a shape of (batch_size, sequence_length, 1).\n",
    "\n",
    "Then the `expand(last_hidden_state.size())` then expands this tensor to match the size of the last_hidden_state tensor. The resulting tensor is thus of shape (batch_size, sequence_length, hidden_size).\n",
    "\n",
    "\n",
    "float() is used to ensure that the expanded mask tensor is a floating point tensor, which is necessary for the upcoming multiplication operation"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "38f975b8",
   "metadata": {},
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Create Model </h3>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "39984a2d",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:05.432174Z",
     "iopub.status.busy": "2022-06-01T14:01:05.431981Z",
     "iopub.status.idle": "2022-06-01T14:01:05.439378Z",
     "shell.execute_reply": "2022-06-01T14:01:05.438779Z"
    },
    "papermill": {
     "duration": 0.049371,
     "end_time": "2022-06-01T14:01:05.440993",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.391622",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "class TextModel(nn.Module):\n",
    "    def __init__(self, model_name):\n",
    "        super(TextModel, self).__init__()\n",
    "        self.model = AutoModel.from_pretrained(model_name)\n",
    "        self.config = AutoConfig.from_pretrained(model_name)\n",
    "        self.drop = nn.Dropout(p=0.2)\n",
    "        self.pooler = MeanPooling()\n",
    "        self.fc = nn.Linear(self.config.hidden_size, CONFIG['num_classes'])\n",
    "        \n",
    "    def forward(self, ids, mask):        \n",
    "        out = self.model(input_ids=ids,attention_mask=mask,\n",
    "                         output_hidden_states=False)\n",
    "        out = self.pooler(out.last_hidden_state, mask)\n",
    "        out = self.drop(out)\n",
    "        outputs = self.fc(out)\n",
    "        return outputs"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "72789004",
   "metadata": {},
   "source": [
    "### Why we need `nn.Linear(self.config.hidden_size, CONFIG['num_classes'])`\n",
    "\n",
    "The purpose of this line is to create the final layer of the neural network model that will map the transformer's output vectors (of size hidden_size) to logits for each of the classes in the classification problem (of size num_classes).\n",
    "\n",
    "Each logit can be thought of as a raw prediction score for a class. To turn these logits into probabilities, they can be passed through a softmax function. The class with the highest probability can be chosen as the model's prediction.\n",
    "\n",
    "This linear layer is crucial because the transformer model by itself outputs feature vectors that represent the input text in a high-dimensional space, but does not perform any task-specific classification. The linear layer takes these high-dimensional representations and maps them to a space that corresponds to the classes we're trying to predict.\n",
    "\n",
    "In summary, this line of code is defining the \"decision-making\" part of the model, where the \"knowledge\" learned and encoded by the transformer model is used to make a final prediction for the task at hand."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "faeff787",
   "metadata": {
    "papermill": {
     "duration": 0.039339,
     "end_time": "2022-06-01T14:01:05.519382",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.480043",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Loss Function </h3>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6dba6634",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:05.599210Z",
     "iopub.status.busy": "2022-06-01T14:01:05.598995Z",
     "iopub.status.idle": "2022-06-01T14:01:05.602483Z",
     "shell.execute_reply": "2022-06-01T14:01:05.601823Z"
    },
    "papermill": {
     "duration": 0.045192,
     "end_time": "2022-06-01T14:01:05.604227",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.559035",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def criterion(outputs, labels):\n",
    "    return nn.CrossEntropyLoss()(outputs, labels)\n",
    "\"\"\" nn.CrossEntropyLoss()(outputs, labels) is creating an instance of the nn.CrossEntropyLoss class and\n",
    "immediately calling it as a function with the outputs and labels as arguments. \"\"\""
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "48a7f3de",
   "metadata": {
    "papermill": {
     "duration": 0.039351,
     "end_time": "2022-06-01T14:01:05.682697",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.643346",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Training Function </h3>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1ab9a499",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:05.762357Z",
     "iopub.status.busy": "2022-06-01T14:01:05.762145Z",
     "iopub.status.idle": "2022-06-01T14:01:05.771949Z",
     "shell.execute_reply": "2022-06-01T14:01:05.771237Z"
    },
    "papermill": {
     "duration": 0.0517,
     "end_time": "2022-06-01T14:01:05.773605",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.721905",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def train_one_epoch(model, optimizer, scheduler, dataloader, device, epoch):\n",
    "    model.train()\n",
    "    \n",
    "    dataset_size = 0\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n",
    "    \"\"\" The total argument in tqdm specifies the total number of iterations (or updates to the progress bar). In this case, len(dataloader) is used as the total which is the total number of batches in the dataloader. \"\"\"\n",
    "    for step, data in bar:\n",
    "        ids = data['input_ids'].to(device, dtype = torch.long)\n",
    "        mask = data['attention_mask'].to(device, dtype = torch.long)\n",
    "        targets = data['target'].to(device, dtype=torch.long)\n",
    "        \n",
    "        batch_size = ids.size(0)\n",
    "\n",
    "        outputs = model(ids, mask)\n",
    "        \n",
    "        loss = criterion(outputs, targets)\n",
    "        \n",
    "        \"\"\" Gradient accumulation involves accumulating gradients over multiple mini-batches before performing a weight update step. \n",
    "        And that Gradient accumulation over several forward passes is achieved through the following two lines in the train_one_epoch() function: \"\"\"\n",
    "        loss = loss / CONFIG['n_accumulate']\n",
    "        \"\"\" The `backward()` call on the next line calculates the gradients of the loss with respect to model parameters. Importantly, these gradients are not removed after the computation, they remain stored in the .grad attributes of the model parameters.\n",
    "        \n",
    "        BUT Instead of updating the parameters right away, add the computed gradients to the accumulated gradients. This step is repeated for a specified number of mini-batches.   \"\"\"\n",
    "        loss.backward()\n",
    "    \n",
    "        # After accumulating gradients over the desired number of mini-batches, perform the weight update step. \n",
    "        if (step + 1) % CONFIG['n_accumulate'] == 0:\n",
    "            # performs the actual parameter update using the accumulated gradients.\n",
    "            optimizer.step()\n",
    "\n",
    "            #  clears out all the accumulated gradients from the parameters to prepare for the next round of accumulation. This happens after every CONFIG['n_accumulate'] batches, as checked by the if condition.\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            if scheduler is not None:\n",
    "                scheduler.step()\n",
    "                \n",
    "        running_loss += (loss.item() * batch_size)\n",
    "        dataset_size += batch_size\n",
    "        \n",
    "        epoch_loss = running_loss / dataset_size\n",
    "        \n",
    "        bar.set_postfix(Epoch=epoch, Train_Loss=epoch_loss,\n",
    "                        LR=optimizer.param_groups[0]['lr'])\n",
    "    gc.collect()\n",
    "    \n",
    "    return epoch_loss"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "36780164",
   "metadata": {},
   "source": [
    "### Explanation for `train_one_epoch`\n",
    "\n",
    "\n",
    "The dataloader object is responsible for serving batches of data. It also handles shuffling of the data and parallelism in the loading of the data onto the device (e.g., a GPU).\n",
    "\n",
    "When enumerate is applied to dataloader, it generates pairs (step, data), where step is the count of batches that have been loaded so far and data is the batch data at the current step.\n",
    "\n",
    "The output of **`tqdm(enumerate(dataloader), total=len(dataloader))`** is not a typical data structure like a list, dictionary, or DataFrame, which you can directly inspect or print out. Instead, it's an iterator wrapped with tqdm progress bar functionality.\n",
    "\n",
    "When you use this iterator in a loop, like:\n",
    "\n",
    "\n",
    "```\n",
    "for step, data in tqdm(enumerate(dataloader), total=len(dataloader)):    \n",
    "\n",
    "```\n",
    "\n",
    "\n",
    "In each iteration, it outputs a tuple, (step, data), where:\n",
    "\n",
    "**step**: is the index of the current batch (starts from 0 and goes up to len(dataloader) - 1).\n",
    "\n",
    "**data**: is the actual content of the batch. This is typically a dictionary where each key is a name of a field in your dataset and the value is a tensor of batched values for that field. For instance, in your script, 'input_ids', 'attention_mask', and 'target' are expected keys in the data."
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "a879f75d",
   "metadata": {},
   "source": [
    "-------------------------\n",
    "\n",
    "### Gradient accumulation - Explanation of line `if (step + 1) % CONFIG['n_accumulate'] == 0:`\n",
    "\n",
    "To understand gradient accumulation, let's first review the standard training procedure for deep learning models. In the standard approach, a mini-batch of training samples is fed into the model, and the gradients of the model parameters with respect to the loss function are computed using backpropagation. Then, these gradients are used to update the model's parameters using an optimization algorithm, such as stochastic gradient descent (SGD) or Adam.\n",
    "\n",
    "In gradient accumulation, instead of updating the model's parameters after each mini-batch, we accumulate gradients over multiple mini-batches before performing the weight update step.\n",
    "\n",
    "**The main steps involved in gradient accumulation are as follows:**\n",
    "\n",
    "Initialize the gradients: Before starting the training loop, the gradients for all model parameters are initialized to zero.\n",
    "\n",
    "Accumulate gradients: For each mini-batch, compute the gradients ( with `loss.backward()` ) of the model parameters with respect to the loss function using backpropagation. \n",
    "\n",
    "BUT Instead of updating the parameters right away, add the computed gradients to the accumulated gradients. This step is repeated for a specified number of mini-batches.\n",
    "\n",
    "Weight update step: After accumulating gradients over the desired number of mini-batches, perform the weight update step. With `optimizer.step()` This involves updating the model's parameters using the accumulated gradients. The update can be done using any optimization algorithm, such as SGD or Adam.\n",
    "\n",
    "Reset gradients: After the weight update step, reset the accumulated gradients to zero to prepare for the next iteration. With `optimizer.zero_grad()`\n",
    "\n",
    "------------------------------------\n",
    "\n",
    "## In the above  `train_one_epoch()` method in which line exactly we sum these gradients over several forward passes ?\n",
    "\n",
    "Gradient accumulation over several forward passes is achieved through the following two lines in the train_one_epoch() function:\n",
    "\n",
    "```py\n",
    "loss = loss / CONFIG['n_accumulate']\n",
    "loss.backward()\n",
    "\n",
    "```\n",
    "\n",
    "In the first line, the loss for the current mini-batch is divided by CONFIG['n_accumulate']. This effectively scales down the gradient that will be computed in the next step. This is necessary because later we are summing (or rather, accumulating) CONFIG['n_accumulate'] of these gradients.\n",
    "\n",
    "#### The `backward()` call on the next line calculates the gradients of the loss with respect to model parameters. Importantly, these gradients are not removed after the computation, they remain stored in the .grad attributes of the model parameters.\n",
    "\n",
    "#### The key point is that every time backward() is called, gradients are computed and then added to whatever is currently stored in the .grad attributes of the parameters. Therefore, if we call backward() on loss / CONFIG['n_accumulate'] for CONFIG['n_accumulate'] mini-batches, the gradients stored in the .grad attributes of the parameters end up being the sum of the gradients for those mini-batches.\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "27a16f5b",
   "metadata": {
    "papermill": {
     "duration": 0.039232,
     "end_time": "2022-06-01T14:01:05.852067",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.812835",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Validation Function </h3>\n",
    "</div>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "037d6962",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:05.933153Z",
     "iopub.status.busy": "2022-06-01T14:01:05.932932Z",
     "iopub.status.idle": "2022-06-01T14:01:05.940756Z",
     "shell.execute_reply": "2022-06-01T14:01:05.939966Z"
    },
    "papermill": {
     "duration": 0.05106,
     "end_time": "2022-06-01T14:01:05.942702",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.891642",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "@torch.no_grad()\n",
    "def valid_one_epoch(model, dataloader, device, epoch):\n",
    "    model.eval()\n",
    "    \n",
    "    dataset_size = 0\n",
    "    running_loss = 0.0\n",
    "    \n",
    "    bar = tqdm(enumerate(dataloader), total=len(dataloader))\n",
    "    for step, data in bar:        \n",
    "        ids = data['input_ids'].to(device, dtype = torch.long)\n",
    "        mask = data['attention_mask'].to(device, dtype = torch.long)\n",
    "        targets = data['target'].to(device, dtype=torch.long)\n",
    "        \n",
    "        batch_size = ids.size(0)\n",
    "\n",
    "        outputs = model(ids, mask)\n",
    "        \n",
    "        loss = criterion(outputs, targets)\n",
    "        \n",
    "        running_loss += (loss.item() * batch_size)\n",
    "        dataset_size += batch_size\n",
    "        \n",
    "        epoch_loss = running_loss / dataset_size\n",
    "        \n",
    "        bar.set_postfix(Epoch=epoch, Valid_Loss=epoch_loss,\n",
    "                        LR=optimizer.param_groups[0]['lr'])   \n",
    "    \n",
    "    gc.collect()\n",
    "    \n",
    "    return epoch_loss"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "41bdc2c6",
   "metadata": {
    "papermill": {
     "duration": 0.039659,
     "end_time": "2022-06-01T14:01:06.021918",
     "exception": false,
     "start_time": "2022-06-01T14:01:05.982259",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "\n",
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Run Training </h3>\n",
    "</div>\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2d584bdc",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:06.102973Z",
     "iopub.status.busy": "2022-06-01T14:01:06.102755Z",
     "iopub.status.idle": "2022-06-01T14:01:06.113624Z",
     "shell.execute_reply": "2022-06-01T14:01:06.112805Z"
    },
    "papermill": {
     "duration": 0.05265,
     "end_time": "2022-06-01T14:01:06.115368",
     "exception": false,
     "start_time": "2022-06-01T14:01:06.062718",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def run_training(model, optimizer, scheduler, train_loader, valid_loader, device, num_epochs, fold):\n",
    "    \"\"\"\n",
    "    Train and validate a PyTorch model for a specified number of epochs.\n",
    "\n",
    "    Parameters:\n",
    "    model (torch.nn.Module): The model to train.\n",
    "    optimizer (torch.optim.Optimizer): The optimizer for the model.\n",
    "    scheduler (torch.optim.lr_scheduler): The learning rate scheduler.\n",
    "    train_loader (torch.utils.data.DataLoader): The DataLoader for the training data.\n",
    "    valid_loader (torch.utils.data.DataLoader): The DataLoader for the validation data.\n",
    "    device (str): The device to train on (\"cpu\" or \"cuda\").\n",
    "    num_epochs (int): The number of epochs to train for.\n",
    "    fold (int): The fold number in k-fold cross-validation.\n",
    "\n",
    "    Returns:\n",
    "    model (torch.nn.Module): The trained model.\n",
    "    history (dict): A dictionary containing the training and validation loss for each epoch.\n",
    "    \"\"\"\n",
    "    # To automatically log gradients\n",
    "    wandb.watch(model, log_freq=100)\n",
    "    \n",
    "    if torch.cuda.is_available():\n",
    "        print(\"[INFO] Using GPU: {}\\n\".format(torch.cuda.get_device_name()))\n",
    "    \n",
    "    start = time.time()\n",
    "    \n",
    "    # Store the initial state of the model\n",
    "    best_model_wts = copy.deepcopy(model.state_dict())\n",
    "    best_epoch_loss = np.inf\n",
    "    \n",
    "    # Store the loss for each epoch\n",
    "    history = defaultdict(list)\n",
    "    \n",
    "    for epoch in range(1, num_epochs + 1): \n",
    "        gc.collect()\n",
    "        train_epoch_loss = train_one_epoch(model, optimizer, scheduler, \n",
    "                                           dataloader=train_loader, \n",
    "                                           device=CONFIG['device'], epoch=epoch)\n",
    "        \n",
    "        val_epoch_loss = valid_one_epoch(model, valid_loader, device=CONFIG['device'], \n",
    "                                         epoch=epoch)\n",
    "    \n",
    "        history['Train Loss'].append(train_epoch_loss)\n",
    "        history['Valid Loss'].append(val_epoch_loss)\n",
    "        \n",
    "        wandb.log({\"Train Loss\": train_epoch_loss})\n",
    "        wandb.log({\"Valid Loss\": val_epoch_loss})\n",
    "        \n",
    "        # If the validation loss improved, save the model weights\n",
    "        if val_epoch_loss <= best_epoch_loss:\n",
    "            print(f\"{b_}Validation Loss Improved ({best_epoch_loss} ---> {val_epoch_loss})\")\n",
    "            best_epoch_loss = val_epoch_loss\n",
    "            run.summary[\"Best Loss\"] = best_epoch_loss\n",
    "            best_model_wts = copy.deepcopy(model.state_dict())\n",
    "            PATH = f\"Loss-Fold-{fold}.bin\"\n",
    "            torch.save(model.state_dict(), PATH)\n",
    "            # Save a model file from the current directory\n",
    "            print(f\"Model Saved{sr_}\")\n",
    "            \n",
    "        print()\n",
    "    \n",
    "    end = time.time()\n",
    "    \n",
    "     # Print total training time and best validation loss\n",
    "    time_elapsed = end - start\n",
    "    print('Training complete in {:.0f}h {:.0f}m {:.0f}s'.format(\n",
    "        time_elapsed // 3600, (time_elapsed % 3600) // 60, (time_elapsed % 3600) % 60))\n",
    "    print(\"Best Loss: {:.4f}\".format(best_epoch_loss))\n",
    "    \n",
    "    # load best model weights\n",
    "    model.load_state_dict(best_model_wts)\n",
    "    \n",
    "    return model, history"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "686238ec",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:06.196573Z",
     "iopub.status.busy": "2022-06-01T14:01:06.195973Z",
     "iopub.status.idle": "2022-06-01T14:01:06.202362Z",
     "shell.execute_reply": "2022-06-01T14:01:06.201671Z"
    },
    "papermill": {
     "duration": 0.049043,
     "end_time": "2022-06-01T14:01:06.204069",
     "exception": false,
     "start_time": "2022-06-01T14:01:06.155026",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_dataloader(fold):\n",
    "    \"\"\"\n",
    "    Returns train and validation data loaders for a given fold.\n",
    "\n",
    "    Args:\n",
    "        fold (int): The fold number for which to create the data loaders.\n",
    "\n",
    "    Returns:\n",
    "        Tuple[torch.utils.data.DataLoader, torch.utils.data.DataLoader]: The train and validation data loaders.\n",
    "    \"\"\"\n",
    "    df_train = df[df.kfold != fold].reset_index(drop=True)\n",
    "    df_valid = df[df.kfold == fold].reset_index(drop=True)\n",
    "    \n",
    "    train_dataset = TextDataset(df_train, tokenizer=CONFIG['tokenizer'], max_length=CONFIG['max_length'])\n",
    "    valid_dataset = TextDataset(df_valid, tokenizer=CONFIG['tokenizer'], max_length=CONFIG['max_length'])\n",
    "\n",
    "    train_loader = DataLoader(train_dataset, batch_size=CONFIG['train_batch_size'], collate_fn=collate_fn, \n",
    "                              num_workers=2, shuffle=True, pin_memory=True, drop_last=True)\n",
    "    valid_loader = DataLoader(valid_dataset, batch_size=CONFIG['valid_batch_size'], collate_fn=collate_fn,\n",
    "                              num_workers=2, shuffle=False, pin_memory=True)\n",
    "    \n",
    "    return train_loader, valid_loader"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6b656405",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:06.284279Z",
     "iopub.status.busy": "2022-06-01T14:01:06.284075Z",
     "iopub.status.idle": "2022-06-01T14:01:06.289469Z",
     "shell.execute_reply": "2022-06-01T14:01:06.288677Z"
    },
    "papermill": {
     "duration": 0.04771,
     "end_time": "2022-06-01T14:01:06.291487",
     "exception": false,
     "start_time": "2022-06-01T14:01:06.243777",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from typing import Optional\n",
    "from torch.optim import Optimizer\n",
    "from torch.optim.lr_scheduler import (   \n",
    "    _LRScheduler\n",
    ")\n",
    "\n",
    "def get_lr_scheduler(optimizer: Optimizer) -> Optional[_LRScheduler]:\n",
    "    \"\"\"\n",
    "    Returns a learning rate scheduler based on the specified configuration.\n",
    "\n",
    "    Args:\n",
    "        optimizer (torch.optim.Optimizer): The optimizer for which to get the scheduler.\n",
    "\n",
    "    Returns:\n",
    "        torch.optim.lr_scheduler._LRScheduler or None: The learning rate scheduler or None if no scheduler is specified.\n",
    "    \"\"\"\n",
    "    scheduler_type = CONFIG.get('scheduler')\n",
    "\n",
    "    if scheduler_type == 'CosineAnnealingLR':\n",
    "        scheduler = lr_scheduler.CosineAnnealingLR(optimizer, T_max=CONFIG.get('T_max'), eta_min=CONFIG.get('min_lr'))\n",
    "    \n",
    "    elif scheduler_type == 'CosineAnnealingWarmRestarts':\n",
    "        scheduler = lr_scheduler.CosineAnnealingWarmRestarts(optimizer, T_0=CONFIG.get('T_0'), eta_min=CONFIG.get('min_lr'))\n",
    "    \n",
    "    elif scheduler_type is None:\n",
    "        return None\n",
    "    \n",
    "    else:\n",
    "        raise ValueError(f\"Invalid scheduler specified: {scheduler_type}\")\n",
    "\n",
    "    return scheduler\n"
   ]
  },
  {
   "attachments": {},
   "cell_type": "markdown",
   "id": "0ae14a83",
   "metadata": {
    "papermill": {
     "duration": 0.038602,
     "end_time": "2022-06-01T14:01:06.369796",
     "exception": false,
     "start_time": "2022-06-01T14:01:06.331194",
     "status": "completed"
    },
    "tags": []
   },
   "source": [
    "<div style=\"background: linear-gradient(45deg, #FFC300, #FF5733, #C70039, #900C3F); padding: 10px; border-radius: 5px; display: flex; align-items: center;\">\n",
    "    <h3 style=\"font-weight: bold; color: white; margin: 0 auto;\"> Start Training </h3>\n",
    "</div>\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "87e4f8d3",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2022-06-01T14:01:06.450866Z",
     "iopub.status.busy": "2022-06-01T14:01:06.449353Z",
     "iopub.status.idle": "2022-06-01T21:00:12.718914Z",
     "shell.execute_reply": "2022-06-01T21:00:12.718153Z"
    },
    "papermill": {
     "duration": 25146.312237,
     "end_time": "2022-06-01T21:00:12.720971",
     "exception": false,
     "start_time": "2022-06-01T14:01:06.408734",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "for fold in range(0, CONFIG['n_fold']):\n",
    "    print(f\"{y_}====== Fold: {fold} ======{sr_}\")\n",
    "    run = wandb.init(project='E_commerce_Review', \n",
    "                     config=CONFIG,\n",
    "                     job_type='Train',\n",
    "                     group=CONFIG['group'],\n",
    "                     tags=[CONFIG['model_name'], f'{HASH_NAME}'],\n",
    "                     name=f'{HASH_NAME}-fold-{fold}',\n",
    "                     anonymous='must')\n",
    "    \n",
    "    train_loader, valid_loader = get_dataloader(fold=fold)\n",
    "    \n",
    "    model = TextModel(CONFIG['model_name'])\n",
    "    model.to(CONFIG['device'])\n",
    "    \n",
    "    optimizer = AdamW(model.parameters(), lr=CONFIG['learning_rate'], weight_decay=CONFIG['weight_decay'])\n",
    "    scheduler = get_lr_scheduler(optimizer)\n",
    "    \n",
    "    model, history = run_training(model, optimizer, scheduler, train_loader, valid_loader,\n",
    "                                  device=CONFIG['device'],\n",
    "                                  num_epochs=CONFIG['epochs'],\n",
    "                                  fold=fold, )\n",
    "    \n",
    "    run.finish()\n",
    "    \n",
    "    del model, history, train_loader, valid_loader\n",
    "    _ = gc.collect()\n",
    "    print()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.9"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 25372.109482,
   "end_time": "2022-06-01T21:02:51.634429",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2022-06-01T13:59:59.524947",
   "version": "2.3.4"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {
     "030f8b7a5de24b8a8385a64fd7d9124a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "LabelModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "LabelModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "LabelView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e77a73cb2f394578adb28d23dcc57ea1",
       "placeholder": "​",
       "style": "IPY_MODEL_b5b3e436fbfe4224b7d510a3e59831bf",
       "value": "0.387 MB of 0.387 MB uploaded (0.000 MB deduped)\r"
      }
     },
     "07475a36a55f443297c4ca5e060c8b51": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "LabelModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "LabelModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "LabelView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_162de73a7d704faf855b39d03a5bece7",
       "placeholder": "​",
       "style": "IPY_MODEL_4ac6999fc4164d23a8b270584d9004e7",
       "value": ""
      }
     },
     "07ec3251544f47ef9e60e5d918de4fd0": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_890685b163304e40bee7d770de99ee86",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_b25a92ec8e564c99bf54daa3af1ffa6a",
       "value": 0
      }
     },
     "111d10a59f534889813b43b76932304a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "162de73a7d704faf855b39d03a5bece7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "172ad66f2ac3443aa6bf3d9c2637035b": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "174dd2253cac499a978974e91612dc49": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1909d15b482e48e2b63e20fab66b3e66": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "1ccc6a3490264cc2b6802c30c5d18d40": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7b6a6d342ca741058c1976a507394e03",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_59527cb15d124e6fb649ae9b9ce438de",
       "value": 0
      }
     },
     "1ea8322b545c49b5a613bb540aa8eec8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7dc6a3c0cc054b678ed96fbf68d095bf",
       "placeholder": "​",
       "style": "IPY_MODEL_857e5d98a6f547159d4da0a31417b117",
       "value": "Downloading: 100%"
      }
     },
     "23d85a2a43f041a0975efeda912443ad": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "VBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "VBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_ceb8f89cb5854f159b28bde96fbe8bfe",
        "IPY_MODEL_1ccc6a3490264cc2b6802c30c5d18d40"
       ],
       "layout": "IPY_MODEL_df8e4c4f834a47e7be7c869017c458b0"
      }
     },
     "24f637dcdaa54f8dbe0522b60eab7152": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "2827e56aa4f84ce7b159ec6d1ea14654": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_e6a2f250fe97496c82cb4fe9cef870f9",
       "placeholder": "​",
       "style": "IPY_MODEL_e7d544f58ec04951b0384be60a211e05",
       "value": " 579/579 [00:00&lt;00:00, 23.2kB/s]"
      }
     },
     "2e7f09a5ea4c4f3f899231fc39748081": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "33bbcda301154b25a4cedc674884a5e5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "36e22c9a08c748968b1498bd5b9997f3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_be9e1b9030fd4bb790bcc624bf622d69",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_2e7f09a5ea4c4f3f899231fc39748081",
       "value": 1
      }
     },
     "3786323c92d643d3a647300b51b2e64d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7724cbde417444dab7e4ee6461ec39bb",
       "max": 52,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_a76fe953621b42dd8053a4f272e0882e",
       "value": 52
      }
     },
     "37aca71e92a842eaaef5a3d7b7b64644": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "388a157173b744ed8070d634f1951b6c": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_9d18350564f843d68871694306e6ebf1",
        "IPY_MODEL_46bd1acb379840d9a5f86b6ab7784117",
        "IPY_MODEL_ea7ff23034b3430599ff2ff247440b7a"
       ],
       "layout": "IPY_MODEL_d53f629b44644a17b0bb3f5fc4e1b304"
      }
     },
     "3dd02f3122d5415b80150c1e429c89d3": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "40cf44c0703e4c5f9936c68a56cb17da": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "41d0a81aa3c8495cb9f705c39bacef65": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "4465a70d070a43108a251210c30f2a42": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_1909d15b482e48e2b63e20fab66b3e66",
       "placeholder": "​",
       "style": "IPY_MODEL_40cf44c0703e4c5f9936c68a56cb17da",
       "value": "Downloading: 100%"
      }
     },
     "46bd1acb379840d9a5f86b6ab7784117": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8d99639de819424893c86066665f982b",
       "max": 2464616,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_9f8f912f31f04c46bf7b093dc36efe3a",
       "value": 2464616
      }
     },
     "4797499274864576b3fe82a0c717b708": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4ac6999fc4164d23a8b270584d9004e7": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "4e867cec08964d0d9862cc0d2b5355e6": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "5326937e5c4b4c1dba6ffbd0a2633563": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "VBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "VBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_68edc33982784516bd01af54db5bc4d4",
        "IPY_MODEL_07ec3251544f47ef9e60e5d918de4fd0"
       ],
       "layout": "IPY_MODEL_ac4a9a34a9dd4265b36ca8eaebe25b6c"
      }
     },
     "555db773bfb54e1dbf5ab6e8d8787b4e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "59527cb15d124e6fb649ae9b9ce438de": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "5ab023d17b3645b7b34f29daeca754e8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_8e41955831d443c0a8bf3b77afcb91d4",
       "max": 371146213,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_ab7bc3aa9f404cd3a5ac8a0f5413a86d",
       "value": 371146213
      }
     },
     "5e66f7f74035469f924afdacd2ab3954": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_4465a70d070a43108a251210c30f2a42",
        "IPY_MODEL_5ab023d17b3645b7b34f29daeca754e8",
        "IPY_MODEL_984a5eb19588495c95a3c46fdc0a1e2e"
       ],
       "layout": "IPY_MODEL_a65054caf04146608b7082d9ab5fc1d2"
      }
     },
     "68edc33982784516bd01af54db5bc4d4": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "LabelModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "LabelModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "LabelView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_a013596da3b64e61a57bcf593ecfff6d",
       "placeholder": "​",
       "style": "IPY_MODEL_ab47e9978c6a45d59f03690568b996a3",
       "value": ""
      }
     },
     "6d8cd376df0d425aa88aa95a0e4fe94e": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "763257b4322f4ee08b328dce68ca2796": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7b4ea72a2ffc4ce09e0a8e9a809bfcf7",
       "placeholder": "​",
       "style": "IPY_MODEL_111d10a59f534889813b43b76932304a",
       "value": "Downloading: 100%"
      }
     },
     "7724cbde417444dab7e4ee6461ec39bb": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7b4ea72a2ffc4ce09e0a8e9a809bfcf7": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7b6a6d342ca741058c1976a507394e03": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7bb860151f6741758b7d6a3b2a3278fd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "7dc6a3c0cc054b678ed96fbf68d095bf": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "80a0ef388bd54492b4896c772c6aad78": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "857e5d98a6f547159d4da0a31417b117": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "865fd003613e457190bf12253aba98c5": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "877fcb64505f474487482bbc5004a932": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "890685b163304e40bee7d770de99ee86": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8c96e1e14c0b46d7a2387f168b3b6524": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "8d953cd985614c8e80f0b1c81a0ef991": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_763257b4322f4ee08b328dce68ca2796",
        "IPY_MODEL_de567006c6b945a9927a3a1378782921",
        "IPY_MODEL_2827e56aa4f84ce7b159ec6d1ea14654"
       ],
       "layout": "IPY_MODEL_33bbcda301154b25a4cedc674884a5e5"
      }
     },
     "8d99639de819424893c86066665f982b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "8e41955831d443c0a8bf3b77afcb91d4": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "958cb79cebb04c6bb48384104673c8d9": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "VBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "VBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_030f8b7a5de24b8a8385a64fd7d9124a",
        "IPY_MODEL_d11cc68a7de74dada6bbe7b578d4dcb5"
       ],
       "layout": "IPY_MODEL_a74f546a74d94231b864ea013ef2bf12"
      }
     },
     "966e83ece3e348cda0141f24ce97fd37": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "LabelModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "LabelModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "LabelView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_96e1aa1784504ab9bf7ae9583577ba6a",
       "placeholder": "​",
       "style": "IPY_MODEL_37aca71e92a842eaaef5a3d7b7b64644",
       "value": "0.389 MB of 0.389 MB uploaded (0.000 MB deduped)\r"
      }
     },
     "96e1aa1784504ab9bf7ae9583577ba6a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "984a5eb19588495c95a3c46fdc0a1e2e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_bf5d6096a1bf4658aadc87f9683d520f",
       "placeholder": "​",
       "style": "IPY_MODEL_b76d3e6c43124c6b8b9006c04e0e3e76",
       "value": " 354M/354M [00:19&lt;00:00, 23.4MB/s]"
      }
     },
     "9896533da5474a12a9455c62e99dafd5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_1ea8322b545c49b5a613bb540aa8eec8",
        "IPY_MODEL_3786323c92d643d3a647300b51b2e64d",
        "IPY_MODEL_b4e3b3ffe3244438a87afcbb1c9ccd3e"
       ],
       "layout": "IPY_MODEL_e1546df6b42b4a6eb6d0a764d1b93ce9"
      }
     },
     "98bbbd34e3224129afcf77e657d163fd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9c26c0a5ce8347518a66a3a89383d734": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "LabelModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "LabelModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "LabelView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_4e867cec08964d0d9862cc0d2b5355e6",
       "placeholder": "​",
       "style": "IPY_MODEL_a83aefd7f2304540894c285d2872711d",
       "value": "0.387 MB of 0.387 MB uploaded (0.000 MB deduped)\r"
      }
     },
     "9d18350564f843d68871694306e6ebf1": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_f8e22863d6104042ad0ac5cb9cd5c51b",
       "placeholder": "​",
       "style": "IPY_MODEL_80a0ef388bd54492b4896c772c6aad78",
       "value": "Downloading: 100%"
      }
     },
     "9ebf04cc8bdc49f4a9c3c12f8bb500bd": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "9f8f912f31f04c46bf7b093dc36efe3a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "a013596da3b64e61a57bcf593ecfff6d": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a65054caf04146608b7082d9ab5fc1d2": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a74f546a74d94231b864ea013ef2bf12": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "a76fe953621b42dd8053a4f272e0882e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "a83aefd7f2304540894c285d2872711d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ab47e9978c6a45d59f03690568b996a3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ab7bc3aa9f404cd3a5ac8a0f5413a86d": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "ac4a9a34a9dd4265b36ca8eaebe25b6c": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b1f22c0758ea46139589facc4170eb32": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "b25a92ec8e564c99bf54daa3af1ffa6a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "ProgressStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "ProgressStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "bar_color": null,
       "description_width": ""
      }
     },
     "b4e3b3ffe3244438a87afcbb1c9ccd3e": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_c643eb9b15f84f8eb089d1704d96576a",
       "placeholder": "​",
       "style": "IPY_MODEL_4797499274864576b3fe82a0c717b708",
       "value": " 52.0/52.0 [00:00&lt;00:00, 1.97kB/s]"
      }
     },
     "b5b3e436fbfe4224b7d510a3e59831bf": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "b76d3e6c43124c6b8b9006c04e0e3e76": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "be9e1b9030fd4bb790bcc624bf622d69": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "bf5d6096a1bf4658aadc87f9683d520f": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "c643eb9b15f84f8eb089d1704d96576a": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "ceb8f89cb5854f159b28bde96fbe8bfe": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "LabelModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "LabelModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "LabelView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_b1f22c0758ea46139589facc4170eb32",
       "placeholder": "​",
       "style": "IPY_MODEL_ff190fe8489242a78054e5056bbf0e80",
       "value": ""
      }
     },
     "d11cc68a7de74dada6bbe7b578d4dcb5": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_7bb860151f6741758b7d6a3b2a3278fd",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_555db773bfb54e1dbf5ab6e8d8787b4e",
       "value": 1
      }
     },
     "d53f629b44644a17b0bb3f5fc4e1b304": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "dc3b71504aa84476826fdd2e5ee982e8": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "VBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "VBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_07475a36a55f443297c4ca5e060c8b51",
        "IPY_MODEL_f97b91eb89104e18ac879f2b7bb82533"
       ],
       "layout": "IPY_MODEL_3dd02f3122d5415b80150c1e429c89d3"
      }
     },
     "de567006c6b945a9927a3a1378782921": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "success",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_9ebf04cc8bdc49f4a9c3c12f8bb500bd",
       "max": 579,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_8c96e1e14c0b46d7a2387f168b3b6524",
       "value": 579
      }
     },
     "df8e4c4f834a47e7be7c869017c458b0": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e1546df6b42b4a6eb6d0a764d1b93ce9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e63822dd68b94b87a359d60682aacd97": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "VBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "VBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_9c26c0a5ce8347518a66a3a89383d734",
        "IPY_MODEL_f83343b1e3b6437a8a824d4fdf1403b3"
       ],
       "layout": "IPY_MODEL_174dd2253cac499a978974e91612dc49"
      }
     },
     "e6a2f250fe97496c82cb4fe9cef870f9": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e77a73cb2f394578adb28d23dcc57ea1": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "e7d544f58ec04951b0384be60a211e05": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     },
     "ea7ff23034b3430599ff2ff247440b7a": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "HTMLModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "HTMLModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "HTMLView",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_865fd003613e457190bf12253aba98c5",
       "placeholder": "​",
       "style": "IPY_MODEL_172ad66f2ac3443aa6bf3d9c2637035b",
       "value": " 2.35M/2.35M [00:01&lt;00:00, 3.19MB/s]"
      }
     },
     "f40745d5f8fb4896b3ea4d74cbe12640": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "VBoxModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "VBoxModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "VBoxView",
       "box_style": "",
       "children": [
        "IPY_MODEL_966e83ece3e348cda0141f24ce97fd37",
        "IPY_MODEL_36e22c9a08c748968b1498bd5b9997f3"
       ],
       "layout": "IPY_MODEL_6d8cd376df0d425aa88aa95a0e4fe94e"
      }
     },
     "f83343b1e3b6437a8a824d4fdf1403b3": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_41d0a81aa3c8495cb9f705c39bacef65",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_877fcb64505f474487482bbc5004a932",
       "value": 1
      }
     },
     "f8e22863d6104042ad0ac5cb9cd5c51b": {
      "model_module": "@jupyter-widgets/base",
      "model_module_version": "1.2.0",
      "model_name": "LayoutModel",
      "state": {
       "_model_module": "@jupyter-widgets/base",
       "_model_module_version": "1.2.0",
       "_model_name": "LayoutModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "LayoutView",
       "align_content": null,
       "align_items": null,
       "align_self": null,
       "border": null,
       "bottom": null,
       "display": null,
       "flex": null,
       "flex_flow": null,
       "grid_area": null,
       "grid_auto_columns": null,
       "grid_auto_flow": null,
       "grid_auto_rows": null,
       "grid_column": null,
       "grid_gap": null,
       "grid_row": null,
       "grid_template_areas": null,
       "grid_template_columns": null,
       "grid_template_rows": null,
       "height": null,
       "justify_content": null,
       "justify_items": null,
       "left": null,
       "margin": null,
       "max_height": null,
       "max_width": null,
       "min_height": null,
       "min_width": null,
       "object_fit": null,
       "object_position": null,
       "order": null,
       "overflow": null,
       "overflow_x": null,
       "overflow_y": null,
       "padding": null,
       "right": null,
       "top": null,
       "visibility": null,
       "width": null
      }
     },
     "f97b91eb89104e18ac879f2b7bb82533": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "FloatProgressModel",
      "state": {
       "_dom_classes": [],
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "FloatProgressModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/controls",
       "_view_module_version": "1.5.0",
       "_view_name": "ProgressView",
       "bar_style": "",
       "description": "",
       "description_tooltip": null,
       "layout": "IPY_MODEL_98bbbd34e3224129afcf77e657d163fd",
       "max": 1,
       "min": 0,
       "orientation": "horizontal",
       "style": "IPY_MODEL_24f637dcdaa54f8dbe0522b60eab7152",
       "value": 0
      }
     },
     "ff190fe8489242a78054e5056bbf0e80": {
      "model_module": "@jupyter-widgets/controls",
      "model_module_version": "1.5.0",
      "model_name": "DescriptionStyleModel",
      "state": {
       "_model_module": "@jupyter-widgets/controls",
       "_model_module_version": "1.5.0",
       "_model_name": "DescriptionStyleModel",
       "_view_count": null,
       "_view_module": "@jupyter-widgets/base",
       "_view_module_version": "1.2.0",
       "_view_name": "StyleView",
       "description_width": ""
      }
     }
    },
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}