task2.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "9RH2W_iWTQOw"
      },
      "source": [
        "## Imports and Installations\n",
        "\n",
        "To tackle this assignement we need to install the **transformers** and **dataset** packages provided by Huggingface and the **allennlp_models** to take advantage of their SQUADf1 implementation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SBz3g9Mb3lDU",
        "outputId": "4e17dab2-fc06-4208-fc9a-a85883a639e5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: allennlp_models in /usr/local/lib/python3.8/dist-packages (2.10.1)\n",
            "Requirement already satisfied: conllu==4.4.2 in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (4.4.2)\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (2.8.0)\n",
            "Requirement already satisfied: py-rouge==1.1 in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (1.1)\n",
            "Requirement already satisfied: allennlp<2.11,>=2.10.1 in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (2.10.1)\n",
            "Requirement already satisfied: torch<1.13.0,>=1.7.0 in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (1.12.1)\n",
            "Requirement already satisfied: nltk>=3.6.5 in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (3.7)\n",
            "Requirement already satisfied: word2number>=1.1 in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (1.1)\n",
            "Requirement already satisfied: ftfy in /usr/local/lib/python3.8/dist-packages (from allennlp_models) (6.1.1)\n",
            "Requirement already satisfied: base58>=2.1.1 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (2.1.1)\n",
            "Requirement already satisfied: sentencepiece>=0.1.96 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.1.97)\n",
            "Requirement already satisfied: cached-path<1.2.0,>=1.1.3 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (1.1.6)\n",
            "Requirement already satisfied: h5py>=3.6.0 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (3.8.0)\n",
            "Requirement already satisfied: traitlets>5.1.1 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (5.7.1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.0.53)\n",
            "Requirement already satisfied: spacy<3.4,>=2.1.0 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (3.3.2)\n",
            "Requirement already satisfied: filelock<3.8,>=3.3 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (3.7.1)\n",
            "Requirement already satisfied: numpy>=1.21.4 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (1.21.6)\n",
            "Requirement already satisfied: scipy>=1.7.3 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (1.7.3)\n",
            "Requirement already satisfied: wandb<0.13.0,>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.12.21)\n",
            "Requirement already satisfied: scikit-learn>=1.0.1 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (1.0.2)\n",
            "Requirement already satisfied: pytest>=6.2.5 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (7.2.1)\n",
            "Requirement already satisfied: typer>=0.4.1 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.4.2)\n",
            "Requirement already satisfied: tensorboardX>=1.2 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (2.5.1)\n",
            "Requirement already satisfied: termcolor==1.1.0 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (1.1.0)\n",
            "Requirement already satisfied: huggingface-hub>=0.0.16 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.10.1)\n",
            "Requirement already satisfied: protobuf<4.0.0,>=3.12.0 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (3.19.6)\n",
            "Requirement already satisfied: tqdm>=4.62 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (4.64.1)\n",
            "Requirement already satisfied: requests>=2.28 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (2.28.2)\n",
            "Requirement already satisfied: lmdb>=1.2.1 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (1.4.0)\n",
            "Requirement already satisfied: dill>=0.3.4 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.3.6)\n",
            "Requirement already satisfied: transformers<4.21,>=4.1 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (4.20.1)\n",
            "Requirement already satisfied: fairscale==0.4.6 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.4.6)\n",
            "Requirement already satisfied: jsonnet>=0.10.0 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.19.1)\n",
            "Requirement already satisfied: torchvision<0.14.0,>=0.8.1 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (0.13.1)\n",
            "Requirement already satisfied: more-itertools>=8.12.0 in /usr/local/lib/python3.8/dist-packages (from allennlp<2.11,>=2.10.1->allennlp_models) (9.0.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.8/dist-packages (from nltk>=3.6.5->allennlp_models) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.8/dist-packages (from nltk>=3.6.5->allennlp_models) (1.2.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.8/dist-packages (from nltk>=3.6.5->allennlp_models) (2022.6.2)\n",
            "Requirement already satisfied: typing-extensions in /usr/local/lib/python3.8/dist-packages (from torch<1.13.0,>=1.7.0->allennlp_models) (4.4.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (1.3.5)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (21.3)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (2022.11.0)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (0.18.0)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (9.0.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (3.8.3)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (6.0)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (3.2.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.8/dist-packages (from datasets->allennlp_models) (0.70.14)\n",
            "Requirement already satisfied: wcwidth>=0.2.5 in /usr/local/lib/python3.8/dist-packages (from ftfy->allennlp_models) (0.2.5)\n",
            "Requirement already satisfied: rich<13.0,>=12.1 in /usr/local/lib/python3.8/dist-packages (from cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (12.6.0)\n",
            "Requirement already satisfied: google-cloud-storage<3.0,>=1.32.0 in /usr/local/lib/python3.8/dist-packages (from cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (2.7.0)\n",
            "Requirement already satisfied: boto3<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (1.26.55)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets->allennlp_models) (1.3.3)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets->allennlp_models) (1.3.1)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets->allennlp_models) (4.0.2)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets->allennlp_models) (2.1.1)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets->allennlp_models) (1.8.2)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets->allennlp_models) (6.0.4)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets->allennlp_models) (22.2.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->datasets->allennlp_models) (3.0.9)\n",
            "Requirement already satisfied: exceptiongroup>=1.0.0rc8 in /usr/local/lib/python3.8/dist-packages (from pytest>=6.2.5->allennlp<2.11,>=2.10.1->allennlp_models) (1.1.0)\n",
            "Requirement already satisfied: tomli>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from pytest>=6.2.5->allennlp<2.11,>=2.10.1->allennlp_models) (2.0.1)\n",
            "Requirement already satisfied: pluggy<2.0,>=0.12 in /usr/local/lib/python3.8/dist-packages (from pytest>=6.2.5->allennlp<2.11,>=2.10.1->allennlp_models) (1.0.0)\n",
            "Requirement already satisfied: iniconfig in /usr/local/lib/python3.8/dist-packages (from pytest>=6.2.5->allennlp<2.11,>=2.10.1->allennlp_models) (2.0.0)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.28->allennlp<2.11,>=2.10.1->allennlp_models) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.28->allennlp<2.11,>=2.10.1->allennlp_models) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.28->allennlp<2.11,>=2.10.1->allennlp_models) (1.26.14)\n",
            "Requirement already satisfied: threadpoolctl>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from scikit-learn>=1.0.1->allennlp<2.11,>=2.10.1->allennlp_models) (3.1.0)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (2.4.5)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.9.0,>=1.7.4 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (1.8.2)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.4.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (0.7.9)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (1.0.4)\n",
            "Requirement already satisfied: thinc<8.1.0,>=8.0.14 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (8.0.17)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (2.0.8)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (3.0.8)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (2.11.3)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (2.0.7)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (57.4.0)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (6.3.0)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.9 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (3.0.11)\n",
            "Requirement already satisfied: pathy>=0.3.5 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (0.10.1)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (1.0.9)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (3.3.0)\n",
            "Requirement already satisfied: wasabi<1.1.0,>=0.9.1 in /usr/local/lib/python3.8/dist-packages (from spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (0.10.1)\n",
            "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /usr/local/lib/python3.8/dist-packages (from torchvision<0.14.0,>=0.8.1->allennlp<2.11,>=2.10.1->allennlp_models) (7.1.2)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.8/dist-packages (from transformers<4.21,>=4.1->allennlp<2.11,>=2.10.1->allennlp_models) (0.12.1)\n",
            "Requirement already satisfied: promise<3,>=2.0 in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (2.3)\n",
            "Requirement already satisfied: psutil>=5.0.0 in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (5.4.8)\n",
            "Requirement already satisfied: pathtools in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (0.1.2)\n",
            "Requirement already satisfied: six>=1.13.0 in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (1.15.0)\n",
            "Requirement already satisfied: setproctitle in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (1.3.2)\n",
            "Requirement already satisfied: docker-pycreds>=0.4.0 in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (0.4.0)\n",
            "Requirement already satisfied: GitPython>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (3.1.30)\n",
            "Requirement already satisfied: sentry-sdk>=1.0.0 in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (1.14.0)\n",
            "Requirement already satisfied: shortuuid>=0.5.0 in /usr/local/lib/python3.8/dist-packages (from wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (1.0.11)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets->allennlp_models) (2022.7)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets->allennlp_models) (2.8.2)\n",
            "Requirement already satisfied: jmespath<2.0.0,>=0.7.1 in /usr/local/lib/python3.8/dist-packages (from boto3<2.0,>=1.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (1.0.1)\n",
            "Requirement already satisfied: botocore<1.30.0,>=1.29.55 in /usr/local/lib/python3.8/dist-packages (from boto3<2.0,>=1.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (1.29.55)\n",
            "Requirement already satisfied: s3transfer<0.7.0,>=0.6.0 in /usr/local/lib/python3.8/dist-packages (from boto3<2.0,>=1.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (0.6.0)\n",
            "Requirement already satisfied: gitdb<5,>=4.0.1 in /usr/local/lib/python3.8/dist-packages (from GitPython>=1.0.0->wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (4.0.10)\n",
            "Requirement already satisfied: google-resumable-media>=2.3.2 in /usr/local/lib/python3.8/dist-packages (from google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (2.4.0)\n",
            "Requirement already satisfied: google-auth<3.0dev,>=1.25.0 in /usr/local/lib/python3.8/dist-packages (from google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (2.16.0)\n",
            "Requirement already satisfied: google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5 in /usr/local/lib/python3.8/dist-packages (from google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (2.11.0)\n",
            "Requirement already satisfied: google-cloud-core<3.0dev,>=2.3.0 in /usr/local/lib/python3.8/dist-packages (from google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (2.3.2)\n",
            "Requirement already satisfied: commonmark<0.10.0,>=0.9.0 in /usr/local/lib/python3.8/dist-packages (from rich<13.0,>=12.1->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (0.9.1)\n",
            "Requirement already satisfied: pygments<3.0.0,>=2.6.0 in /usr/local/lib/python3.8/dist-packages (from rich<13.0,>=12.1->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (2.6.1)\n",
            "Requirement already satisfied: MarkupSafe>=0.23 in /usr/local/lib/python3.8/dist-packages (from jinja2->spacy<3.4,>=2.1.0->allennlp<2.11,>=2.10.1->allennlp_models) (2.0.1)\n",
            "Requirement already satisfied: smmap<6,>=3.0.1 in /usr/local/lib/python3.8/dist-packages (from gitdb<5,>=4.0.1->GitPython>=1.0.0->wandb<0.13.0,>=0.10.0->allennlp<2.11,>=2.10.1->allennlp_models) (5.0.0)\n",
            "Requirement already satisfied: googleapis-common-protos<2.0dev,>=1.56.2 in /usr/local/lib/python3.8/dist-packages (from google-api-core!=2.0.*,!=2.1.*,!=2.2.*,!=2.3.0,<3.0.0dev,>=1.31.5->google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (1.58.0)\n",
            "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /usr/local/lib/python3.8/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (5.2.1)\n",
            "Requirement already satisfied: pyasn1-modules>=0.2.1 in /usr/local/lib/python3.8/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (0.2.8)\n",
            "Requirement already satisfied: rsa<5,>=3.1.4 in /usr/local/lib/python3.8/dist-packages (from google-auth<3.0dev,>=1.25.0->google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (4.9)\n",
            "Requirement already satisfied: google-crc32c<2.0dev,>=1.0 in /usr/local/lib/python3.8/dist-packages (from google-resumable-media>=2.3.2->google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (1.5.0)\n",
            "Requirement already satisfied: pyasn1<0.5.0,>=0.4.6 in /usr/local/lib/python3.8/dist-packages (from pyasn1-modules>=0.2.1->google-auth<3.0dev,>=1.25.0->google-cloud-storage<3.0,>=1.32.0->cached-path<1.2.0,>=1.1.3->allennlp<2.11,>=2.10.1->allennlp_models) (0.4.8)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: transformers in /usr/local/lib/python3.8/dist-packages (4.20.1)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.13,>=0.11.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.12.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.8/dist-packages (from transformers) (4.64.1)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.8/dist-packages (from transformers) (2.28.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.1.0 in /usr/local/lib/python3.8/dist-packages (from transformers) (0.10.1)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from transformers) (3.7.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0,>=0.1.0->transformers) (4.4.0)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2022.12.7)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (1.26.14)\n",
            "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.8/dist-packages (from requests->transformers) (2.1.1)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: datasets in /usr/local/lib/python3.8/dist-packages (2.8.0)\n",
            "Requirement already satisfied: fsspec[http]>=2021.11.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (2022.11.0)\n",
            "Requirement already satisfied: multiprocess in /usr/local/lib/python3.8/dist-packages (from datasets) (0.70.14)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from datasets) (21.3)\n",
            "Requirement already satisfied: huggingface-hub<1.0.0,>=0.2.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.10.1)\n",
            "Requirement already satisfied: responses<0.19 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.18.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (6.0)\n",
            "Requirement already satisfied: pandas in /usr/local/lib/python3.8/dist-packages (from datasets) (1.3.5)\n",
            "Requirement already satisfied: xxhash in /usr/local/lib/python3.8/dist-packages (from datasets) (3.2.0)\n",
            "Requirement already satisfied: aiohttp in /usr/local/lib/python3.8/dist-packages (from datasets) (3.8.3)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.8/dist-packages (from datasets) (1.21.6)\n",
            "Requirement already satisfied: requests>=2.19.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (2.28.2)\n",
            "Requirement already satisfied: dill<0.3.7 in /usr/local/lib/python3.8/dist-packages (from datasets) (0.3.6)\n",
            "Requirement already satisfied: tqdm>=4.62.1 in /usr/local/lib/python3.8/dist-packages (from datasets) (4.64.1)\n",
            "Requirement already satisfied: pyarrow>=6.0.0 in /usr/local/lib/python3.8/dist-packages (from datasets) (9.0.0)\n",
            "Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (6.0.4)\n",
            "Requirement already satisfied: async-timeout<5.0,>=4.0.0a3 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (4.0.2)\n",
            "Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (22.2.0)\n",
            "Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.1)\n",
            "Requirement already satisfied: charset-normalizer<3.0,>=2.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (2.1.1)\n",
            "Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.3.3)\n",
            "Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.8/dist-packages (from aiohttp->datasets) (1.8.2)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (4.4.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.8/dist-packages (from huggingface-hub<1.0.0,>=0.2.0->datasets) (3.7.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->datasets) (3.0.9)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2.10)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (1.26.14)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.8/dist-packages (from requests>=2.19.0->datasets) (2022.12.7)\n",
            "Requirement already satisfied: pytz>=2017.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2022.7)\n",
            "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.8/dist-packages (from pandas->datasets) (2.8.2)\n",
            "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.8/dist-packages (from python-dateutil>=2.7.3->pandas->datasets) (1.15.0)\n"
          ]
        }
      ],
      "source": [
        "!pip install allennlp_models\n",
        "!pip install transformers\n",
        "!pip install datasets\n",
        "import datasets\n",
        "import random\n",
        "import matplotlib.pyplot as plt\n",
        "import matplotlib\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import os\n",
        "from tensorflow import keras\n",
        "from tensorflow.keras.utils import set_random_seed\n",
        "import sklearn as sk\n",
        "from sklearn.model_selection import GroupShuffleSplit\n",
        "from transformers import TFEncoderDecoderModel, AutoTokenizer, DataCollatorForSeq2Seq, TFAutoModel\n",
        "from transformers import Seq2SeqTrainingArguments, Seq2SeqTrainer\n",
        "from tqdm.notebook import tqdm\n",
        "import absl.logging\n",
        "from allennlp_models.rc.tools import squad\n",
        "absl.logging.set_verbosity(absl.logging.ERROR)\n",
        "matplotlib.rcParams['figure.figsize'] = [8, 8]"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We'll now set a seed for results reproducibility"
      ],
      "metadata": {
        "id": "zircwvCrFc_i"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "id": "2GeeM9e_zf07"
      },
      "outputs": [],
      "source": [
        "def set_seed(seed):\n",
        "  np.random.seed(seed)\n",
        "  set_random_seed(seed)\n",
        "  random.seed(seed)\n",
        "  os.environ['TF_DETERMINISTIC_OPS'] = '1'\n",
        "\n",
        "seeds = [42, 2022, 1337]\n",
        "set_seed(seeds[0])"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "51ITlivc3lDW"
      },
      "source": [
        "### Dataset download"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 3,
      "metadata": {
        "id": "XLEUWN8T3lDX"
      },
      "outputs": [],
      "source": [
        "#Code provided in assignment docs\n",
        "#Dataset download\n",
        "import os\n",
        "import urllib.request\n",
        "from tqdm import tqdm\n",
        "\n",
        "class DownloadProgressBar(tqdm):\n",
        "    def update_to(self, b=1, bsize=1, tsize=None):\n",
        "        if tsize is not None:\n",
        "            self.total = tsize\n",
        "        self.update(b * bsize - self.n)\n",
        "        \n",
        "def download_url(url, output_path):\n",
        "    with DownloadProgressBar(unit='B', unit_scale=True,\n",
        "                             miniters=1, desc=url.split('/')[-1]) as t:\n",
        "        urllib.request.urlretrieve(url, filename=output_path, reporthook=t.update_to)\n",
        "\n",
        "def download_data(data_path, url_path, suffix):    \n",
        "    if not os.path.exists(data_path):\n",
        "        os.makedirs(data_path)\n",
        "        \n",
        "    data_path = os.path.join(data_path, f'{suffix}.json')\n",
        "    if not os.path.exists(data_path):\n",
        "        print(f\"Downloading CoQA {suffix} data split... (it may take a while)\")\n",
        "        download_url(url=url_path, output_path=data_path)\n",
        "        print(\"Download completed!\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 4,
      "metadata": {
        "id": "NXeIOoEG3lDY"
      },
      "outputs": [],
      "source": [
        "# Train and test data loading\n",
        "train_url = \"https://nlp.stanford.edu/data/coqa/coqa-train-v1.0.json\"\n",
        "download_data(data_path='coqa', url_path=train_url, suffix='train')\n",
        "\n",
        "test_url = \"https://nlp.stanford.edu/data/coqa/coqa-dev-v1.0.json\"\n",
        "download_data(data_path='coqa', url_path=test_url, suffix='test')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Q3-ADW1m3lDY"
      },
      "source": [
        "### Reading data from JSON"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 5,
      "metadata": {
        "id": "r8sd1knL3lDZ"
      },
      "outputs": [],
      "source": [
        "def from_json(path, columns):\n",
        "    df = pd.read_json(path)\n",
        "    del df[\"version\"]\n",
        "    row_array = []\n",
        "\n",
        "    for i, row in df.iterrows():\n",
        "        for index in range(len(row[\"data\"][\"questions\"])):\n",
        "            row_insert = []\n",
        "            row_insert.append(row[\"data\"][\"source\"])\n",
        "            row_insert.append(row[\"data\"][\"id\"])\n",
        "            row_insert.append(row[\"data\"][\"filename\"])\n",
        "            row_insert.append(row[\"data\"][\"story\"])\n",
        "            row_insert.append(row[\"data\"][\"questions\"][index][\"input_text\"])\n",
        "            row_insert.append(row[\"data\"][\"questions\"][index][\"turn_id\"])\n",
        "            row_insert.append(row[\"data\"][\"answers\"][index][\"span_start\"])\n",
        "            row_insert.append(row[\"data\"][\"answers\"][index][\"span_end\"])\n",
        "            row_insert.append(row[\"data\"][\"answers\"][index][\"span_text\"])\n",
        "            row_insert.append(row[\"data\"][\"answers\"][index][\"input_text\"])\n",
        "            row_insert.append(row[\"data\"][\"answers\"][index][\"turn_id\"])\n",
        "            row_array.append(row_insert[:len(columns)])\n",
        "\n",
        "    df = pd.DataFrame(row_array, columns=columns)\n",
        "    df = df[df[\"answer\"] != \"unknown\"]  # Removing unanserable questions\n",
        "    return df"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 6,
      "metadata": {
        "id": "IwwyHxke3lDa"
      },
      "outputs": [],
      "source": [
        "columns_headers = [\"source\", \"id\", \"filename\", \"story\", \"question\", \"qturn_id\", \"span_start\",\"span_end\", \"span_text\", \"answer\", \"aturn_id\"]\n",
        "train_val_df = from_json('coqa/train.json', columns_headers)\n",
        "test_df = from_json('coqa/test.json', columns_headers) "
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ssTYMCxi3lDc"
      },
      "source": [
        "### Splitting the train set into train and validation"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "2E8bsZpE3lDd",
        "outputId": "ff93f816-e579-40cf-f1cf-a959c0f0d458"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "      source                              id             filename  \\\n",
              "0  wikipedia  3zotghdk5ibi9cex97fepx7jetpso7  Vatican_Library.txt   \n",
              "1  wikipedia  3zotghdk5ibi9cex97fepx7jetpso7  Vatican_Library.txt   \n",
              "2  wikipedia  3zotghdk5ibi9cex97fepx7jetpso7  Vatican_Library.txt   \n",
              "3  wikipedia  3zotghdk5ibi9cex97fepx7jetpso7  Vatican_Library.txt   \n",
              "4  wikipedia  3zotghdk5ibi9cex97fepx7jetpso7  Vatican_Library.txt   \n",
              "\n",
              "                                               story  \\\n",
              "0  The Vatican Apostolic Library (), more commonl...   \n",
              "1  The Vatican Apostolic Library (), more commonl...   \n",
              "2  The Vatican Apostolic Library (), more commonl...   \n",
              "3  The Vatican Apostolic Library (), more commonl...   \n",
              "4  The Vatican Apostolic Library (), more commonl...   \n",
              "\n",
              "                            question  qturn_id  span_start  span_end  \\\n",
              "0  When was the Vat formally opened?         1         151       179   \n",
              "1           what is the library for?         2         454       494   \n",
              "2                 for what subjects?         3         457       511   \n",
              "3                               and?         4         457       545   \n",
              "4          what was started in 2014?         5         769       879   \n",
              "\n",
              "                                           span_text  \\\n",
              "0                       Formally established in 1475   \n",
              "1           he Vatican Library is a research library   \n",
              "2  Vatican Library is a research library for hist...   \n",
              "3  Vatican Library is a research library for hist...   \n",
              "4  March 2014, the Vatican Library began an initi...   \n",
              "\n",
              "                                answer  aturn_id  \n",
              "0  It was formally established in 1475         1  \n",
              "1                             research         2  \n",
              "2                     history, and law         3  \n",
              "3     philosophy, science and theology         4  \n",
              "4                           a  project         5  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e8918cf2-b0ee-41ae-8c94-8ef2dba39ee8\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>source</th>\n",
              "      <th>id</th>\n",
              "      <th>filename</th>\n",
              "      <th>story</th>\n",
              "      <th>question</th>\n",
              "      <th>qturn_id</th>\n",
              "      <th>span_start</th>\n",
              "      <th>span_end</th>\n",
              "      <th>span_text</th>\n",
              "      <th>answer</th>\n",
              "      <th>aturn_id</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>wikipedia</td>\n",
              "      <td>3zotghdk5ibi9cex97fepx7jetpso7</td>\n",
              "      <td>Vatican_Library.txt</td>\n",
              "      <td>The Vatican Apostolic Library (), more commonl...</td>\n",
              "      <td>When was the Vat formally opened?</td>\n",
              "      <td>1</td>\n",
              "      <td>151</td>\n",
              "      <td>179</td>\n",
              "      <td>Formally established in 1475</td>\n",
              "      <td>It was formally established in 1475</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>wikipedia</td>\n",
              "      <td>3zotghdk5ibi9cex97fepx7jetpso7</td>\n",
              "      <td>Vatican_Library.txt</td>\n",
              "      <td>The Vatican Apostolic Library (), more commonl...</td>\n",
              "      <td>what is the library for?</td>\n",
              "      <td>2</td>\n",
              "      <td>454</td>\n",
              "      <td>494</td>\n",
              "      <td>he Vatican Library is a research library</td>\n",
              "      <td>research</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>wikipedia</td>\n",
              "      <td>3zotghdk5ibi9cex97fepx7jetpso7</td>\n",
              "      <td>Vatican_Library.txt</td>\n",
              "      <td>The Vatican Apostolic Library (), more commonl...</td>\n",
              "      <td>for what subjects?</td>\n",
              "      <td>3</td>\n",
              "      <td>457</td>\n",
              "      <td>511</td>\n",
              "      <td>Vatican Library is a research library for hist...</td>\n",
              "      <td>history, and law</td>\n",
              "      <td>3</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>wikipedia</td>\n",
              "      <td>3zotghdk5ibi9cex97fepx7jetpso7</td>\n",
              "      <td>Vatican_Library.txt</td>\n",
              "      <td>The Vatican Apostolic Library (), more commonl...</td>\n",
              "      <td>and?</td>\n",
              "      <td>4</td>\n",
              "      <td>457</td>\n",
              "      <td>545</td>\n",
              "      <td>Vatican Library is a research library for hist...</td>\n",
              "      <td>philosophy, science and theology</td>\n",
              "      <td>4</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>wikipedia</td>\n",
              "      <td>3zotghdk5ibi9cex97fepx7jetpso7</td>\n",
              "      <td>Vatican_Library.txt</td>\n",
              "      <td>The Vatican Apostolic Library (), more commonl...</td>\n",
              "      <td>what was started in 2014?</td>\n",
              "      <td>5</td>\n",
              "      <td>769</td>\n",
              "      <td>879</td>\n",
              "      <td>March 2014, the Vatican Library began an initi...</td>\n",
              "      <td>a  project</td>\n",
              "      <td>5</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e8918cf2-b0ee-41ae-8c94-8ef2dba39ee8')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e8918cf2-b0ee-41ae-8c94-8ef2dba39ee8 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e8918cf2-b0ee-41ae-8c94-8ef2dba39ee8');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ],
      "source": [
        "#creating the group splitter to split the dataset with respect to stories\n",
        "#story cannot be presented in train and validation sets at the same time\n",
        "group_split = GroupShuffleSplit(n_splits=2, test_size=0.2, random_state=42)\n",
        "split = group_split.split(train_val_df, groups=train_val_df[\"id\"])\n",
        "\n",
        "train_inds, val_inds = next(split)\n",
        "\n",
        "train_df = train_val_df.iloc[train_inds]\n",
        "val_df = train_val_df.iloc[val_inds]\n",
        "\n",
        "train_df.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "JbvnyTmj4mRG"
      },
      "source": [
        "# Dataset Statistics\n",
        "\n",
        "We'll now extract useful statistics from the dataset.\n",
        "\n",
        "The first thing I want to check is how the length of the question + the context (our input) is distributed in the training set:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 8,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "id": "x6d286ro4ls0",
        "outputId": "c75c5d8b-9628-40f9-aea7-e60f3bdfad21"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAhkAAAHwCAYAAAALuX1cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeZhkZX33//e399n3YZuNHRE3HEXBBQUVjUselwjG3WjyUx81mieaaIyaVZ88JibBhUSDGneMigZBEJeIG6CAMywywMAszPRMT8/W+3L//jinh5qmu6eqp2vrer+uq6+us9Spb1WdOvWp+9znnEgpIUmSNNOaql2AJEmanQwZkiSpLAwZkiSpLAwZkiSpLAwZkiSpLAwZkiSpLAwZZRQR/ysitkTEwYh4XBHznx8RWytRWyVExLqISBHRUoXHfmpE3DXF9JJqi4jLI+KvZ67C4kXE5oi4cIaWVdI6qcqKiN+PiO+VYblV3bZUc1tQy/LP4UkVfswPRMR/Vurxjhgy8g1cX/5i7Mw3tvMrUVwxKv2ClegfgLemlOanlH49fmL+oTulCnXNeiml/0kpnT42PJNf1OVUgTAz5TpZDjP5nKoZ9mbaRF+8KaUvpJSeXc26ZkK9fN5m0nSCXP45vLdcNdWCYlsyXpBSmg+cDawH3lfKg0SmEVtN1gIbq12EVKCh1sk8lLy22nVodrOFZgoppSn/gM3AhQXD/xf4Tn77ScBPgb3ArcD5BfP9EPgb4AagDzgFeCRwLbAH2An8eT5vE/Ae4B6gC/gqsDSftg5IwGuAB4DdwHvzaRcBg8AQcBC4NR//OuAO4ABwL/CH457TnwIPAtuBP8iXf0o+rZ3s194DeY2fBOZM8to0kQWu+4FO4HPAonwZB/Pl9gD3THDfHxdMPwi8HDgf2Aq8K1/eg8DrCu5TSm2nAD8C9uWv2VcKpn0M2ALsB24Gnlow7QPA14D/zF+/3wCnAX+W17QFePa49/nvgF/my/vWBO9dSz68CPh0/ry2AX8NNB+p3nHP67PAu/LbJ+TLf0s+fDLZutU09lrm4z8PjJKthwfz93+stoetV5M87uXAXxcMPx+4hWzd/ynw6HGfmT8Bbsufz1eAjiOtf8CbyNblwbzObxezvJlaJ/P7T/YZbQf+Ka95e367PZ92PpOst1M8p+OBrwO7gPuAt+Xjl+bLekE+PB/YBLx6smUdYft1OfDaI82Xz/uq/HXrAt5LwbZvgvf/fPL1a6rnk097InAT2edjJ/DRfPwD+ftxMP97MvBa4CcF9z0XuDF/328Ezh332fsrsm3sAeB7wPJJnttM1NtBtl3oIlvvbwSOmeCxSv685Y/5s3y5DwL/CrQVTE/AHwF35/NcCsQkz3XKZU0w/1N46Htsy9j6whTbWx5a598N7CDbZvblz3vs/Ty+yOc19t1zef68/jt/P38BnDxJzd8la5EsHHcr8OIit/H/OdF6UbC9GVvvJ/1uLvavpJABrCb7FfRXZBv4LuB5eSHPyodXFHwAHiDbaLUAC/IX+V1kK+sC4Jx83rcDPwdW5W/sp4Av5dPW5W/EvwFzgMcAA8Ajxr9gBTX/DtkXTgBPB3qBs/NpF+UrxSOBuWQfmsI3+h+BK8k2dguAbwN/N8lr83qyDeBJZBvD/wI+P9EKNMn9D5uev+HDwIeA1vy17QWWTKO2L5FtKJvy1/spBdNeCSzL35d35a9HR8Hr2Q88J5/+ObKN0Hvzmt4I3DduQ7cNOAuYR7bhGluBx967sZDxjfy9nQesJAsmf3ikeid4zce+qF5BtvJ/pWDatybZqG7m8LA8VtuE69UEj3s5+ZcM8DiyL9NzgGayDedmHvrS3Zw/t+Pz9+oO4I+KXP8OPc642idc3kyuk0z9Gf0Q2Wd0JbCCbKP8V0Wut4c9p/w9vhl4P9CW13ov8Jx8+rPz12hl/v5cMdH7UNQGrsiQAZxJ9sXwNLJt0Efz53TEkFHE8/kZ8Kr89nzgSRN9PvJxryUPGfl73U0WflqAS/LhZQWfvXvIfgTMyYf/fpLnNxP1/iHZNmcu2Xr/eGDhkb43ityOP57sR2tLPu8dwDvGrbffARYDa8jC0UWTPPaUyxo371qyL/RLyNbdZcBjj7S95aF1/sNk68scJv7CLuZ5FX72u8iCSQvwBeDLk9T9auCGcevvXh7aBh1pG19syJj0u7noz2ARH77NZB++vWQp/+P5C/puCjZe+bzXAK8p+AB8qGDaJcCvJ3mMO4ALCoaPI/vFMvbGJGBVwfRfAhePf8GmeA7fBN6e3/4MBV/MZL8gU/4/yH7lnVww/ckUfKmOW+73gTcXDJ8+Vvf4FWiS+08UMvo4fKPTma+kpdb2OeCywtdtijq6gccUvJ7XFkx7Qf7+j7U4LMjrXlzwPv99wfxnkv3SbC5471qAY8g2KnMK5r0E+EEp9ZKFx26yDeUnyTZ8YxvPzwLvnOjDw+QbvQnXqwke93IeChmfIP+CLZh+F/D0gsd6ZcG0jwCfPNL6N/5xxtU+4fJmcp1k6s/oPcDzCoafA2w+0no70XMiC2cPjFv+nwH/UTD8L2StaNvIv1Qne32OsL5cTnEh4/0UbNDJgvAgxYWMKZ8PWavlBxnXysCRQ8argF+Ou8/PeOiX9g+B9xVMezNw9STPbybqfT3jWu2meD03c3Sft3cA3ygYThz+Q+mrwHuKXAcOW9YEz/th0zjC9jZ/PQc5vIXy0GtcbC08/LP/7wXTngfcOclyFuT1rc2H/wb4zBSPO34bX2zImPS7uZjXPqVUdJ+M300pLU4prU0pvTml1EeWAF8WEXvH/sianY4ruN+WgturyTZUE1kLfKNgOXcAI2RfTGN2FNzuJUvYE4qI50bEzyNiT7685wHL88nHj6ur8PYKspR+c0EtV+fjJ3I8WfAacz8PfaFOV1dKabhgeOy5llrbn5J9UH4ZERsj4vVjEyLiTyLijojYly9nEQ+9PpA1DY7pA3anlEYKhuHw17/wNbyf7BdB4fIge49bgQcL6v8U2a/VKestlFK6h+zD9VjgqWS/brZHxOlkrVY/mvjlmFTR69W45/Kucev+arL14UjLnWr9m4k6j2adnOozOtFyC5/vZOvtRNYCx497/f58XI2XkbWOXZ5S6iqi9kMi4raC5b4C+HjBY318krsd9r6klHrIflUW40jP5w1krQ13RsSNEfH8Ipc7/jUnHz6hYHi66+906v082Q/JL0fE9oj4SES0Fvlcpqw3Ik6LiO9ExI6I2A/8LQ/fhhT1XItc1pjJ1vlitre7Ukr9kyx3OrVAkc8xpXSAbLfKxfmoS8haPsYe90jb+GIV8908paPprLKFrCXjjVPMk8bNf/Ek820BXp9SumH8hIhYd4Q6Ch+DiGgna7J/NVnT+VBEfJPsCwyy5uBVBXdZXXB7N9mX6CNTStuO8LiQ7ZteWzC8hqwJbefEsx+VkmpLKe0g27VBRDwFuC4ifkwWAv8UuADYmFIajYhuHnp9pqPwNVxDlnR3jxu/hawlY/m4L6Mp600pbZrg8X4EvJRs3+a2iPgR2S6LJWT9JCaSJhk/HVuAv0kp/c007jvV+gdHX+fRrJNTfUbHljvWaXRNPq4Y45/TFrJfhKdONHNENJOFjM8Bb46I/yhYD474+qSUHl2wrMuBH6aULj/C3R4EHlFwv7lkzc1jesi+dMYcW3B7yueTUrobuCTv/P5i4IqIWFbEcxn/XkL2ul99hPsdybTqzYPXB4EP5tvlq8ha8D490WJKrOkTwK+BS1JKByLiHWSf8ekoZVlbyHZPjFfM9nb8c5zoOc/k8xrvS8Bf5tv1DuAHkB2+T/Hb+MPW6/yzVxikJv1uLtbRHPHxn8ALIuI5EdEcER35ITyrJpn/O8BxEfGOiGiPiAURcU4+7ZPA30TEWoCIWBERLyqyjp3AuoKjV9rI9h3tAoYj4rlk+3jHfBV4XUQ8It+Q/MXYhJTSKNk+w3+MiJV5LSdExHMmeewvAX8cESdGdljv35L1D3jYl+gUtRd1jHSptUXEywrei26yD8AoWTPbMNnr0xIR7wcWFlnvZF4ZEWfmr+eHyPahjxTOkFJ6kKxj2v+LiIUR0RQRJ0fE049Q70R+BLyVrFkXsmbjt5I1M49Mcp+iX+si/BvwRxFxTmTmRcTvRMSCIu476fo3Q3UezTo51Wf0S8D78s/mcrLdC8UeOj7+Of0SOBAR746IOfn246yIeEI+/c/J3v/Xk3U0/1y+8ZtoWTPlCuD5EfGUiGgjW48Lt4+3AM+LiKURcSxZs3dRzyciXhkRK/LP8N78PqNkn8HRKZ7PVcBpEfGKiGiJiJeT7Y78zlE+12nVGxHPiIhH5e/FfrIfE5N9Rkt9nxbkyzwYEWcA/990ntg0lvUF4MKI+L38NV4WEY+dxncBZM95WUQsKtPzGu8qshD6IbLP+Nh7Uco2/rdAR779aiXrNN5eMP1ovpuBowgZKaUtwIvINgi7yBLP/5lsmXnzzrPI9vHvIOsl/Ix88sfIOth8LyIOkHU0OWei5Uzga/n/roj4Vf44byPbmHeTNZdeWVDHd4F/Jkt9m/LHguxXNmR9TTYBP4+sees6sv3aE/kMWRPij8k6R/YD/7vIuiHbN/bZyJqifq+I+Uup7QnALyLiINnzf3vKjse+huyX0G/Jml77Kb7JfjKfJ9ufuIMsUb9tkvleTRYCbyd7b67god1rk9U7kR+RfZDGQsZPyNL4jyeZH7IjYN6Xv9Z/UsRzmlRK6SayVpd/zZ/HJrJ96cXc90jr36eBM/M6vzmN8qa9Th7hM/rXZEcc3EbWV+JX+bhiHPac8iD4fLJdXveR/Wr8d2BRRDweeCfw6ny+D5MFjvdMtKwiH/+IUkobgbcAXyRr1egmO3pgzOfJeu9vJgvLXym476TPJ5/lImBjvm5/jKwfQl9KqZf8CLz8+TxpXE1d+XLfRbbr5k+B56eUdh/lc51WvWStN1eQfWneQfY5/PwkD1Pq5+1PyLbVB8i+3L8y9ewzs6yU0gNku9PfRXZE1S1knVKhtO0tKaU7ycL4vfnzPn6Gn9f4xxsg69h9Idl6O6bobXxKaR9ZX55/J+v/1MPh6/2U382RnT/rqVPVGXlnjoYVEY8ANpD1yi22BUK5iPghWSeif692LfXI9a92RcRm4A9SStdVuxapXjXiCbLGTq3cHhFLyH4pfdsNvCrF9U9So2jIkEF22GMnWa/iEWZ2P5l0JK5/khpCw+8ukSRJ5dGoLRmSJKnMDBmSJKks6vbKccuXL0/r1q2rdhmSJFXEzTffvDulNNlZnmtS3YaMdevWcdNNN1W7DEmSKiIixp9mvua5u0SSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJWFIUOSJJVF2UNGRHwmIjojYsMk0yMi/jkiNkXEbRFxdrlrkiRJ5VeJlozLgYummP5c4NT8703AJypQkyRJKrOWcj9ASunHEbFuilleBHwupZSAn0fE4og4LqX0YLlrk6RGk1JiNMFoSoymRMpvp3SUy52h2mbCzNRy9Mtoa25iTlvz0S+ojpU9ZBThBGBLwfDWfJwhQ1LZpZQYHBllYHiUweFR+gZH6B8aoW9ohL7BEXqHRugfHGF4NB36Yh4dfehL+qEvbBgdfWhcSomR0cTw6EP/h0dGD7s9Nm1oJDEyOpqPH7vPQ8Mjo1mNQ4f+EoPD2bJGUjo8OIxmgWEkHV7L6AyFCRXv5etX8+GXPrraZVRVLYSMokXEm8h2qbBmzZoqVyOpXEZHD//iHxgeYXB4lMGRbDgbV/B/ZJSD/cPs7x9if98QB/qH6RkcpndgJPtfEBwGhsbuO8JAfv9KaWkKWpqDlqYmmpuC1uaguemh4Wxa0NzUVDAt+7+wrZW25qC1uenQX3MTNDcFTTH2B1Fwu7kp8mEOm57d56F5I6Apjv75BUe/kJiBOmZKHGUxpx0zf4YqqV+1EDK2AasLhlfl4x4mpXQZcBnA+vXrzeNSGY39wu8fGmVgaIT+oVH6h0c40D/Mgf4hBoZH81/d2S/r4ZFRhvJf6GO/tsdPH/s1PhYQegaG2dc3xL6+Ifb3D9M7MJwtd3T6H++WpmBBRwvz2luY19bCnLZm5rY1s3hOKx2tzbS3NtHe0kx7S9Pht/O/jtZm5rQ109GS/Z/T1syc1mZam7Mv7ObCL+amsWEO/zJveuh2FiaaDs0jNZJaCBlXAm+NiC8D5wD77I8hTV9KiYHhUXoHR+gZGObgwDC9g8McHMiGD/3l0zsPDHB/V8/Dfv0PDI/OSNN6BLQ2NdHSHLS1NNHW3HTo/7z2FhbNaeXYRR0smtPK3LYW2vIv+7F52lubaR+7z7hp2XAzbS1NLOhoYUFHC3Nam/0yl2pE2UNGRHwJOB9YHhFbgb8EWgFSSp8ErgKeB2wCeoHXlbsmqZ4cHBhm8+4etnb3sqdniL19g+zvG85//We7B/b3D3MgH97XN8TQSHHpoKUpWDKvjROXz2P10rnMb29hbv7Lv6M1+xv7dZ/9ZcFgYUcL7S3NtDZn4WEsRBTefqhJ3y98qVFV4uiSS44wPQFvKXcdUrWllNjXN0Tv4Ai9g1mnwp7BYQ70D7O/b4i9Y7sN+obo7h3kwX393Le7h10HBh62rNbmYNGcVhZ2tLJwTisLO1pYvWQOi+a0sqCjlQUdLYcCw/z2Fua2tzC/vfnQLoR57S3Ma2+mrbnJX/2SyqYWdpdIs0ZKid0HB9na3cvW7j62dvdxf1cP9+3uYVPnQbp6Bo+4jAUdLSyd18bKBe2cf9oKTlwxj5Pyloal89pYPKeNjlbDgaTaZ8iQpulA/xC/2bqPe3f3sHl3D5t2HeTWLXvp7h06bL5l89pYt3wezzxjJacfu4D57WOdEVuY19bMwjlZy8NYK4S7FyTNFoYMqQSdB/r53sadfOe27fzyvj2MHQTR3tLEicvn8awzj+HM4xayeulcVi2ZywlL5jC/3Y+ZpMbk1k86gvu7erj29p1cvWEHNz/QTUpw0op5vPn8U3jiiUs5ZeV8jl3YQZMtEJJ0GEOGNIH+oRG+etMWLr9hM/fu7gHgEcct5B0XnMZFZx3LacfMt0+EJB2BIUMqkFLiylu38+Hv3sn2ff2cvWYxH3jBmTzjjJWsXTav2uVJUl0xZEi5vsER/vwbv+Ebv97GI49fyEde+hjOO2WZLRaSNE2GDDW8/qERrrxlO5f+cBP3d/XyzmedxluecYpHeUjSUTJkqKFt3L6Pt3/5FjZ1HuSMYxfwxTeew7knL692WZI0Kxgy1JBGRxOfueE+PnL1XSye28pnXrueZ5y+0l0jkjSDDBlqODv39/Our97KTzbt5tlnHsPfv+TRLJ3XVu2yJGnWMWSooVyzcQfv/vptDAyN8ncvfhQXP2G1rReSVCaGDDWElBIf+/7d/NN1d/OoExbxTxc/lpNXzK92WZI0qxkyNOsNj4zyp1fcxn/9ehsvOXsVf/fiR9HW0lTtsiRp1jNkaNb78NV38l+/3sYfX3gab7vgFHePSFKFGDI0q333Nw/yb/9zH6958lrefuGp1S5HkhqKbcaatXoGhnnfNzfwmFWLeO/vnFntciSp4diSoVnrP264j66eQT792ifYB0OSqsAtr2alfb1DfOrH93LhI47hsasXV7scSWpIhgzNSp/68T0cHBjmXc8+rdqlSFLDMmRo1tl1YID/uGEzz3/08TziuIXVLkeSGpYhQ7POx3+4icGRUf7Yo0kkqaoMGZpVtu3t4ws/f4CXnH0CJ3lGT0mqKkOGZpV/+f7dJBJvu8BWDEmqNkOGZo3OA/187eatXPLENaxaMrfa5UhSwzNkaNb4xq+2MTKaeM2566pdiiQJQ4ZmiZQSX7t5K2evWezVVSWpRhgyNCvcsmUvmzoP8rL1q6tdiiQpZ8jQrPC1m7fS0drE8x99XLVLkSTlDBmqe/1DI3z71u0896zjWNDRWu1yJEk5Q4bq3jUbd3Cgf5iXPX5VtUuRJBUwZKjufe2mraxaMocnnbSs2qVIkgoYMlTXtu3t44Z7dvOSs1fR1BTVLkeSVMCQobr29Zu3khK81F0lklRzDBmqWyklrrh5K08+aRmrl3qGT0mqNYYM1a2f37uHB/b02oohSTXKkKG69cVfPsDCjhae9yjPjSFJtciQobq068AAV294kJc+fjVz2pqrXY4kaQKGDNWlr928haGRxCvOWVPtUiRJkzBkqO6Mjia++IsHeNJJSzllpRdDk6RaZchQ3bnhnt1s7e7j989ZW+1SJElTMGSo7vz3bQ8yr62ZZ515TLVLkSRNwZChujIymvje7Tt55iOOoaPVDp+SVMsMGaorN27ew56eQZ571rHVLkWSdASGDNWVqzfsoL2liaeftqLapUiSjsCQoboxOpq4esMOnnbaCua1t1S7HEnSERgyVDdu3bqXHfv73VUiSXXCkKG6cfXGHbQ0BRec4VElklQPDBmqCyllu0rOPWU5i+a2VrscSVIRDBmqC3fuOMD9Xb1c9Eh3lUhSvTBkqC58d8MOIvAEXJJURwwZqgvX3b6T9WuXsGJBe7VLkSQVyZChmrdjXz+3P7ifZ5yxstqlSJJKYMhQzfvRbzsBeKYhQ5LqiiFDNe/6Ozs5blEHpx+zoNqlSJJKYMhQTRscHuUnd+/m/NNXEhHVLkeSVAJDhmraTZv30DM44q4SSapDhgzVtB/dvYvW5uDck5dVuxRJUokMGappP7l7N49bs8QLoklSHTJkqGZ1HRxg4/b9PPWU5dUuRZI0DYYM1ayf3tMFwFNONWRIUj0yZKhm/eTu3SzsaOHRqxZXuxRJ0jQYMlSTUkr8ZNNuzj15Oc1NHroqSfXIkKGadN/uHrbt7XNXiSTVMUOGatINm3YD8BQ7fUpS3TJkqCb9cnM3xy3qYO2yudUuRZI0TYYM1ZyUEjfet4f165Z6KnFJqmOGDNWcbXv72LG/nyesW1LtUiRJR8GQoZpz0+ZuANavXVrlSiRJR8OQoZpz4+Y9LGhv4fRjvbS7JNWzioSMiLgoIu6KiE0R8Z4Jpq+JiB9ExK8j4raIeF4l6lJtumlzN2evXeL5MSSpzpU9ZEREM3Ap8FzgTOCSiDhz3GzvA76aUnoccDHw8XLXpdq0r3eIu3YesD+GJM0ClWjJeCKwKaV0b0ppEPgy8KJx8yRgYX57EbC9AnWpBt38wB4A1q+zP4Yk1btKXD/7BGBLwfBW4Jxx83wA+F5E/G9gHnBhBepSDbppczctTcFjvF6JJNW9Wun4eQlweUppFfA84PMR8bDaIuJNEXFTRNy0a9euihep8vvNtn2ccdwC5rQ1V7sUSdJRqkTI2AasLhhelY8r9AbgqwAppZ8BHcDDziedUrospbQ+pbR+xYoVZSpX1ZJSYsO2fZx1/KJqlyJJmgGVCBk3AqdGxIkR0UbWsfPKcfM8AFwAEBGPIAsZNlU0mO37+unuHeKRJxgyJGk2KHvISCkNA28FrgHuIDuKZGNEfCgiXpjP9i7gjRFxK/Al4LUppVTu2lRbNmzbB8BZxy88wpySpHpQiY6fpJSuAq4aN+79BbdvB86rRC2qXRu27aO5KXjEcYYMSZoNaqXjp8SGbfs4deV8Olrt9ClJs4EhQzVjw/b9PNJOn5I0axgyVBM69/ez68AAZ53grhJJmi0MGaoJG7bnnT49skSSZg1DhmrCb7buJwLOtNOnJM0ahgzVhA3b93HS8nnMa6/IAU+SpAowZKgmbNy2z10lkjTLGDJUdV0HB9i+r9/TiUvSLGPIUNVt3L4fgEd6ZIkkzSqGDFXdb/LTiXuODEmaXQwZqrqN2/exZulcFs1prXYpkqQZZMhQ1W3Ytp9H2elTkmYdQ4aqqmdgmAf29HL6sQuqXYokaYYZMlRV9+w6CMBpx8yvciWSpJlmyFBV3b0zCxmnHmNLhiTNNoYMVdXdnQdpbQ7WLp1b7VIkSTPMkKGqunvnAU5aPp+WZldFSZpt3LKrqu7uPMip9seQpFnJkKGq6RscYUt3L6eutD+GJM1GhgxVzT27DpIStmRI0ixlyFDV3N/VC8CJy+dVuRJJUjkYMlQ1m7t6AFi7zCNLJGk2MmSoajbv7mHlgnbmtrVUuxRJUhkYMlQ193f1sm6Zu0okabYyZKhqNnf1sG65u0okabYyZKgqegeH6TwwwFpbMiRp1jJkqCo2786OLHF3iSTNXoYMVcX9HlkiSbOeIUNVsTk/R8Y6z5EhSbOWIUNVcX9XD8vntzO/3cNXJWm2MmSoKjZ39bDOXSWSNKsZMlQVm3f3emSJJM1yhgxVXN/gCDv293Oi58iQpFnNkKGKe2BP1unTlgxJmt0MGaq4sQujeY4MSZrdDBmquM2783NkuLtEkmY1Q4YqbnNXL8vmtbGwo7XapUiSysiQoYq7v6vHM31KUgMwZKjivMS7JDUGQ4Yqqn9ohO37+jyyRJIagCFDFbVlTy8pwTo7fUrSrGfIUEUdujCaLRmSNOsZMlRR93uODElqGIYMVdT9Xb0smtPKorkevipJs50hQxW1pbuX1UvnVLsMSVIFGDJUUVu7+1i12E6fktQIDBmqmJQSW23JkKSGYchQxew+OEj/0CirltiSIUmNwJChitnanR2+umqJLRmS1AgMGaqYrd19AKxeakuGJDUCQ4YqZkveknHCYlsyJKkRGDJUMVu7+1g6r4157S3VLkWSVAGGDFXMlj29rLY/hiQ1DEOGKmZbd59HlkhSAzFkqCJGRxNb9/Z5ZIkkNRBDhipi18EBBodHWeWRJZLUMAwZqohD58jwyBJJahiGDFXE9r39ABxvyJCkhmHIUEU8uC87EddxizuqXIkkqVIMGaqI7Xv7WdDewsKO1mqXIkmqEEOGKmL73j5bMSSpwRgyVBEP7uvnuEX2x5CkRmLIUEU8uK+P423JkKSGYshQ2Q0Mj7D74KAtGZLUYAwZKrsd+7LDV49bZEuGJDWSkkJGRDwlIl6X314RESeWpyzNJp4jQ5IaU9EhIyL+Eng38Gf5qFbgP8tRlGaXQ4fz2ccAACAASURBVOfIsCVDkhpKKS0Z/wt4IdADkFLaDiwoR1GaXR48tLvElgxJaiSlhIzBlFICEkBEzCtPSZpttu/tY+m8Nua0NVe7FElSBZUSMr4aEZ8CFkfEG4HrgH8rT1maTbJzZLirRJIaTdEhI6X0D8AVwNeB04H3p5T+pZj7RsRFEXFXRGyKiPdMMs/vRcTtEbExIr5YbF2qfdv39rmrRJIaUEspM6eUrgWuLeU+EdEMXAo8C9gK3BgRV6aUbi+Y51SyDqXnpZS6I2JlKY+h2rZjfz9PWLe02mVIkiqslKNLDkTE/nF/WyLiGxFx0hR3fSKwKaV0b0ppEPgy8KJx87wRuDSl1A2QUuos9YmoNvUPjbC3d4hj3V0iSQ2nlJaMfyJrifgiEMDFwMnAr4DPAOdPcr8TgC0Fw1uBc8bNcxpARNwANAMfSCldXUJtqlGd+wcAWLmgvcqVSJIqrZSQ8cKU0mMKhi+LiFtSSu+OiD+fgTpOJQsqq4AfR8SjUkp7C2eKiDcBbwJYs2bNUT6kKmHngezw1WMW2pIhSY2mlKNLevPOmU353+8B/fm0NMX9tgGrC4ZX5eMKbQWuTCkNpZTuA35LFjoOk1K6LKW0PqW0fsWKFSWUrmrZud+QIUmNqpSQ8fvAq4BOYGd++5URMQd46xT3uxE4NSJOjIg2st0sV46b55vku1siYjnZ7pN7S6hNNWpnvrvkmIXuLpGkRlP07pKU0r3ACyaZ/JMp7jccEW8FriHrb/GZlNLGiPgQcFNK6cp82rMj4nZgBPg/KaWuYmtT7erc309bSxOL5rRWuxRJUoUVHTIiogN4A/BI4FDbd0rp9Ue6b0rpKuCqcePeX3A7Ae/M/zSL7NzfzzEL24mIapciSaqwUnaXfB44FngO8COyvhUHylGUZo+d+wc4ZoH9MSSpEZUSMk5JKf0F0JNS+izwOzz8UFTpMDsP9NvpU5IaVCkhYyj/vzcizgIWAZ6ZU1Pq3D/ASjt9SlJDKuU8GZdFxBLgfWRHh8wH/qIsVWlWODgwzMGBYVsyJKlBlRIyvp+f9vvHwEkAEXFiWarSrNB56BwZtmRIUiMqZXfJ1ycYd8VMFaLZ59A5Muz4KUkN6YgtGRFxBtlhq4si4sUFkxZScCirNF5nfkrxle4ukaSGVMzuktOB5wOLOfxkXAfIrp4qTWinu0skqaEdMWSklL4FfCsinpxS+lkFatIssXP/AHPbmpnfXkrXH0nSbFHK1n9TfrXVdYX3K+aMn2pMO/Zn58jwbJ+S1JhKCRnfAv4HuI7s+iLSlDr397NygbtKJKlRlRIy5qaU3l22SjTr7Nw/wGNXL652GZKkKinlENbvRMTzylaJZpWU0qGLo0mSGlMpIePtZEGjPyL2R8SBiNhfrsJU3/b3DTMwPOrZPiWpgRW9uySltKCchWh22ek5MiSp4RXdkhGZV0bEX+TDqyPiieUrTfXs0Dky7PgpSQ2rlN0lHweeDLwiHz4IXDrjFWlWOHRKcVsyJKlhlXJ0yTkppbMj4tcAKaXuiGgrU12qc7sOZCFjhS0ZktSwSmnJGIqIZiABRMQKYLQsVanu7ekZoKO1ibltzdUuRZJUJaWEjH8GvgGsjIi/AX4C/G1ZqlLd6+oZZNm8ds/2KUkNrJSjS74QETcDFwAB/G5K6Y6yVaa61nVwkKXz3JsmSY2s6JAREU8CNqaULs2HF0bEOSmlX5StOtWtPT2GDElqdKXsLvkE2RElYw7m46SH2dMzyDJDhiQ1tFJCRqSU0thASmmU0o5OUQPp6hmwJUOSGlwpIePeiHhbRLTmf28H7i1XYapfvYPD9A+NsnS+IUOSGlkpIeOPgHOBbcBW4BzgTeUoSvWt6+AggLtLJKnBFbW7Iz8/xj+mlC4ucz2aBfb0jIUMT8QlSY2sqJaMlNIIsNYzfKoYYyHD3SWS1NhK6bh5L3BDRFwJ9IyNTCl9dMarUl3r6nF3iSSptJBxT/7XBHjZd02q62B23RKPLpGkxlbKGT8/CBARc1NKveUrSfVuT88gbc1NzG/3CGdJamRFH10SEU+OiNuBO/Phx0TEx8tWmepWV362T69bIkmNrZRDWP8JeA7QBZBSuhV4WjmKUn3zlOKSJCgtZJBS2jJu1MgM1qJZoqtnkGUeWSJJDa+UkLElIs4FUn7Gzz8BvAqrHmZPz4BHlkiSSj7j51uAE4DtwGPzYekwew4OstQTcUlSwyvl6JLdwO+XsRbNAv1DI/QMjri7RJJU0tElJ0XEtyNiV0R0RsS3IuKkchan+nPobJ/uLpGkhlfK7pIvAl8FjgOOB74GfKkcRal+jV0czZAhSSolZMxNKX0+pTSc//0n0FGuwlSfunqys33a8VOSVMopGb8bEe8Bvgwk4OXAVRGxFCCltKcM9anOuLtEkjSmlJDxe/n/Pxw3/mKy0GH/DHmZd0nSIaUcXXJiOQvR7NDVM0hLU7BwjtctkaRGV9IZP6Uj2ds7yOK5XrdEkmTI0AzLrlvSWu0yJEk1wJChGdXdO8TiuXb6lCSVdjKu7xczTo1tb+8gSw0ZkiSK6PgZER3AXGB5RCwBxna2LyS7jol0yJ6eIR6/1t0lkqTiji75Q+AdZGf5/FXB+P3Av5ajKNWnlNKhjp+SJB0xZKSUPgZ8LCL+d0rpXypQk+rUgYFhhkeTu0skSUBpHT8/ExHvi4jLACLi1Ih4fpnqUh3a2zMEwOK57i6RJJUYMoBB4Nx8eBvw1zNekepWd292ts8ltmRIkigtZJycUvoIMASQUurloU6gEnvGQobXLZEkUVrIGIyIOWTXKSEiTgYGylKV6tLeQy0Z7i6RJJV2gbS/BK4GVkfEF4DzgNeWoyjVp+68T4ZXYJUkQWkXSLs2In4FPIlsN8nbU0q7y1aZ6k537yBNAQs7bMmQJJV2xs/zgP6U0n8Di4E/j4i1ZatMdae7d5BFc1pparKrjiSptD4ZnwB6I+IxwDuBe4DPlaUq1aXuniE7fUqSDiklZAynlBLwIuDSlNKlwILylKV61N076OGrkqRDSgkZByLiz4BXAv8dEU2AO991SHfvkCFDknRIKSHj5WSHrL4hpbQDWAX837JUpbrU3TPo4auSpENKObpkB/DRguEHsE+GcimlbHeJfTIkSblSji55UkTcGBEHI2IwIkYiYl85i1P96BsaYWB41N0lkqRDStld8q/AJcDdwBzgD4CPl6Mo1Z/u3uxEXO4ukSSNKSVkkFLaBDSnlEZSSv8BXFSeslRvunu8bokk6XClnFa8NyLagFsi4iPAg5QYUjR7eQVWSdJ4pYSEV+XzvxXoAVYDLy5HUao/7i6RJI1XSsj43ZRSf0ppf0rpgymldwLPL1dhqi97vcy7JGmcUkLGayYY99oZqkN1bk/eJ2PxHFsyJEmZI/bJiIhLgFcAJ0bElQWTFgB7ylWY6sve3iEWdLTQ0mw3HUlSppiOnz8l6+S5HPh/BeMPALcV8yARcRHwMaAZ+PeU0t9PMt9LgCuAJ6SUbipm2aoNXrdEkjTeEUNGSul+4H7gydN5gIhoBi4FngVsBW6MiCtTSrePm28B8HbgF9N5HFXXvr4hFtvpU5JU4GjP+Lm/iLs+EdiUUro3pTQIfJnsSq7j/RXwYaC/2JpUO/b1DbHI/hiSpAJHe8bPS4u43wnAloLhrfm4QyLibGB1Sum/S6hHNWRf7xALDRmSpAJVP+Nnfsn4jwLvKmLeN0XETRFx065du472oTWDbMmQJI1XSsg47IyfEfHHRd5/G9mJu8asyseNWQCcBfwwIjYDTwKujIj14xeUUrospbQ+pbR+xYoVJZSuckopGTIkSQ9ztGf8fEkR97sRODUiTsxDysXAoUNhU0r7UkrLU0rrUkrrgJ8DL/TokvrROzjC8GgyZEiSDlP0tUtSSvdHxIr89gdLuN9wRLwVuIbsENbPpJQ2RsSHgJtSSldOvQTVun192SnFPRGXJKlQMSfjCuAvyVowmvJRw8C/pJQ+VMyDpJSuAq4aN+79k8x7fjHLVO0YCxm2ZEiSChWzu+SPgfPITpC1NKW0BDgHOC/vl6EGt7fXkCFJerhiQsargEtSSveNjUgp3Qu8Enh1uQpT/RhryfAQVklSoWJCRmtKaff4kSmlXYDfKmK/u0skSRMoJmQMTnOaGsShPhmeVlySVKCYo0seM8npwwPomOF6VIf29Q3RFDC/reiDlSRJDaCYC6Q1V6IQ1a+xE3E1NUW1S5Ek1ZCSTisuTcSzfUqSJmLI0FHba8iQJE3AkKGjtq/PK7BKkh7OkKGjtt+WDEnSBAwZOmr2yZAkTcSQoaPiZd4lSZMxZOio9AyOMDKaWOyJuCRJ4xgydFS8AqskaTKGDB2Vvb3ZmeUNGZKk8QwZOipegVWSNBlDho6KV2CVJE3GkKGjYp8MSdJkDBk6KoYMSdJkDBk6Kvv6hmhuCua3e5l3SdLhDBk6KmMn4orwMu+SpMMZMnRU9vZ6tk9J0sQMGToqXoFVkjQZQ4aOyr6+IRYbMiRJEzBk6Kjs7R1iidctkSRNwJCho9LdO8jiuW3VLkOSVIMMGZq24ZFRDvQPewVWSdKEDBmatrETcS2xJUOSNAFDhqZtbx4ybMmQJE3EkKFpG7vMu30yJEkTMWRo2vb25i0ZHsIqSZqAIUPT1t1rnwxJ0uQMGZq2sd0li+yTIUmagCFD07a3N7sC68IOr8AqSXo4Q4amrbt30CuwSpImZcjQtO3pGWTZPPtjSJImZsjQtHUdHGTZfEOGJGlihgxN2+6eAZbNa692GZKkGmXI0LTt6bElQ5I0OUOGpmVoZJS9vUMstU+GJGkShgxNS3dPdo6MZfPdXSJJmpghQ9PSlYeM5bZkSJImYcjQtHQdzEKGu0skSZMxZGhaunoGAHeXSJImZ8jQtOw6kIWMFYYMSdIkDBmali17elnQ3sLCOV63RJI0MUOGpmVLdx+rls71uiWSpEkZMjQtD+zpZfWSOdUuQ5JUwwwZKtnA8Aj37e7htGMWVLsUSVINM2SoZJs6DzIymjj9WEOGJGlyhgyV7KebugB43JrFVa5EklTLDBkq2bV37OSMYxewasncapciSaphhgyVpLtnkJs27+FZZx5T7VIkSTXOkKGS/PC3nYwmuPARhgxJ0tQMGSrJdbd3smJBO486YVG1S5Ek1ThDhoo2ODzKj367iwsfsZKmJk/CJUmamiFDRfvFfV0cHBh2V4kkqSiGDBXt+3d00tHaxHmnLK92KZKkOmDIUFFSSlx3x06ecspyOlqbq12OJKkOGDJUlLs7D7K1u48L3FUiSSqSIUNFue6OnQA884yVVa5EklQvDBkqyvV3dPKoExZxzMKOapciSaoThgwd0Z6eQX71QLetGJKkkhgydEQ/vMuzfEqSSmfI0BF9/85OVi5o55HHL6x2KZKkOmLI0JSGRkb58V27eOYZnuVTklQaQ4amdON9ezgwMOyhq5KkkhkyNKXv39lJW0sT552yrNqlSJLqjCFDU7r+zk7OPXkZc9taql2KJKnOGDI0qXt2HeS+3T3uKpEkTUtFQkZEXBQRd0XEpoh4zwTT3xkRt0fEbRHx/YhYW4m6NLXr7+gEPMunJGl6yh4yIqIZuBR4LnAmcElEnDlutl8D61NKjwauAD5S7rp0ZNfdsZMzjl3ACYvnVLsUSVIdqkRLxhOBTSmle1NKg8CXgRcVzpBS+kFKqTcf/DmwqgJ1aQr7eoe46f5uLniErRiSpOmpRMg4AdhSMLw1HzeZNwDfLWtFOqIf3b2LkdFkfwxJ0rTV1CEDEfFKYD3w9Emmvwl4E8CaNWsqWFnj+f4dO1k2r43HrFpc7VIkSXWqEi0Z24DVBcOr8nGHiYgLgfcCL0wpDUy0oJTSZSml9Sml9StWrChLsYLhkVF+eNcuzj99Jc2e5VOSNE2VCBk3AqdGxIkR0QZcDFxZOENEPA74FFnA6KxATZrCzfd3s69vyP4YkqSjUvaQkVIaBt4KXAPcAXw1pbQxIj4UES/MZ/u/wHzgaxFxS0RcOcniVAHX3r6TtuYmnnaarUWSpOmrSJ+MlNJVwFXjxr2/4PaFlahDR5ZS4to7dnLuKcuY315TXXYkSXXGM37qMHd3HuT+rl6edaZHlUiSjo4hQ4e59vadAFzooauSpKNkyNBhvnf7Th6zejHHLOyodimSpDpnyNAh3T2D3LZ1L8883aNKJElHz5ChQ352bxcpwVNOXVbtUiRJs4AhQ4fcsGk389qaebRn+ZQkzQBDhg756T1dnHPSMlqbXS0kSUfPbxMBsG1vH/ft7uG8U5ZXuxRJ0ixhyBCQ7SoBOO8U+2NIkmaGIUMA/HTTbpbPb+P0YxZUuxRJ0ixhyBApJW64p4snn7ycCK+6KkmaGYYMsanzILsODHDeye4qkSTNHEOG+J+7x/pj2OlTkjRzDBniB3d1cvKKeaxeOrfapUiSZhFDRoM70D/Ez+/t4gIviCZJmmGGjAb3k7t3MzSSuOAMr1ciSZpZhowGd90dnSzsaOHxa5dUuxRJ0ixjyGhgo6OJH/22k6efvpIWTyUuSZphfrM0sNu27WP3wUGeecaKapciSZqFDBkN7Po7O4mAp59mfwxJ0swzZDSwH97VyeNWL2bpvLZqlyJJmoUMGQ2q80A/t23dxzM9qkSSVCaGjAb1w7t2AfAMQ4YkqUwMGQ3qB3d2cuzCDs48bmG1S5EkzVKGjAY0ODzK/9y9m2ecscKrrkqSysaQ0YBu2ryHgwPDPON0d5VIksrHkNGArr+zk7bmJq+6KkkqK0NGA/rBXZ2cc9JS5rW3VLsUSdIsZshoMA909XLPrh53lUiSys6Q0WCuv3MngOfHkCSVnSGjwVx/1y5OWj6PdcvnVbsUSdIsZ8hoIL2Dw/z83i5PwCVJqghDRgO5YVMXg8Oj7iqRJFWEIaOBXLNxBws6WnjiiUurXYokqQEYMhrE8Mgo379jJxecsZLWZt92SVL5+W3TIG7c3E137xDPeeSx1S5FktQgDBkN4pqNO2hvaeLpp6+odimSpAZhyGgAo6OJa2/fyVNPXc7cNs/yKUmqDENGA/jl5j1s29vH7zz6uGqXIklqIIaMBvDVG7ewoL2Fix5pyJAkVY4hY5a7v6uHb9+2nf919gnMaWuudjmSpAZiyJjFUkp88Nu309LUxFufcUq1y5EkNRhDxix22Y/v5fo7O/k/zzmdlQs7ql2OJKnBGDJmqZs27+Ej19zF8x51LK87b121y5EkNSBDxix0oH+It3/5Fk5YPIcPv+TRRES1S5IkNSBPmjALffjqO9m+r48r/uhcFnS0VrscSVKDsiVjlvnVA9184RcP8Npz1/H4tUuqXY4kqYEZMmaR4ZFR3vuNDRyzoIN3Pfv0apcjSWpwhoxZ5LM/u587HtzPX77gTOa3uydMklRdhoxZYse+fj76vbt4xukruOgsr7QqSao+Q8Ys8aHvbGR4NPHBF57l0SSSpJpgyJgFvn7zVq76zQ7edsGprFk2t9rlSJIEGDLq3j27DvIX39rAOScu5Y+efnK1y5Ek6RBDRh0bGU288yu30N7SxMcufhzNTe4mkSTVDg9BqGOf/elmbt26j3++5HEcu8hrk0iSaostGXVq294+/uF7d3H+6St4waOPq3Y5kiQ9jCGjDqWU+ItvbiAl+Ovf9WgSSVJtMmTUof/+zYNcf2cn73r2aaxa4tEkkqTaZMioM/t6h/jAlbfzqBMW8dpz11W7HEmSJmXHzzrz91ffQXfvIJe/7gm0NJsRJUm1y2+pOnL3zgN8+cYtvPbcdZx1wqJqlyNJ0pQMGXXkH6/7LfPaWnjLM06pdimSJB2RIaNObNi2j6t+s4PXn7eOpfPaql2OJElHZMioEx+99rcsmtPKHzztpGqXIklSUQwZdeDm+7u5/s5O/vDpJ7Gwo7Xa5UiSVBRDRo1LKfGRq+9k+fw2D1mVJNUVQ0aN++Yt2/jFfXt4+4WnMbfNI44lSfXDkFHD7tpxgPd9YwOPW7OYVzxxTbXLkSSpJIaMGtXdM8gbP3cTc9tb+MTvP97LuEuS6o4howYNjYzyli/+ih37+vnUqx7vZdwlSXWpIiEjIi6KiLsiYlNEvGeC6e0R8ZV8+i8iYl0l6qpFo6OJD357Iz+9p4u/ffGjOHvNkmqXJEnStJS9J2FENAOXAs8CtgI3RsSVKaXbC2Z7A9CdUjolIi4GPgy8vNy1VdvoaGLngX427+5la3cv2/f2c/1dndy6ZS9vfOqJvPTxq6pdoiRJ01aJwxWeCGxKKd0LEBFfBl4EFIaMFwEfyG9fAfxrRERKKVWgvmkbGhmld2CEvqERhkZGGR5NDI+MMjSSGB4dpXdwhL29Q+zrG8z/D7G3b4jdBwa4v6uX+/f00D80etgyT1k5n4+85NG8bL0BQ5JU3yoRMk4AthQMbwXOmWyelNJwROwDlgG7K1Af/+97d/HrB/ZOGBSGRxJDY/8Lxg0OjzI4MnrkhRdoaQoWz21lydw21i6by1NPXc7a5fM4cdk8Vi+dw7GLOmhvaS7Ts5QkqbLq6sQLEfEm4E0Aa9bM3CGdPQMj9A4O09LcREdrEy3tLbQ2By1NTbQ0B63NTTQ3xWHj2lqamN/Wwtz2Fjpam2htbjo0fez/nLZmFs9tZfHcNhbNaWVeWzMRHiUiSWoMlQgZ24DVBcOr8nETzbM1IlqARUDX+AWllC4DLgNYv379jO1Kef8LzpypRUmSpFwlji65ETg1Ik6MiDbgYuDKcfNcCbwmv/1S4Ppa748hSZKmVvaWjLyPxVuBa4Bm4DMppY0R8SHgppTSlcCngc9HxCZgD1kQkSRJdawifTJSSlcBV40b9/6C2/3AyypRiyRJqgzP+ClJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksrCkCFJksoiUkrVrmFaImIXcH8VS1gO7K7i4xfLOmeWdc4s65xZ1jmzaq3OtSmlFdUuohR1GzKqLSJuSimtr3YdR2KdM8s6Z5Z1zizrnFn1Umctc3eJJEkqC0OGJEkqC0PG9F1W7QKKZJ0zyzpnlnXOLOucWfVSZ82yT4YkSSoLWzIkSVJZGDImEBGfiYjOiNhQMG5pRFwbEXfn/5fk4yMi/jkiNkXEbRFxdgXrXB0RP4iI2yNiY0S8vRZrjYiOiPhlRNya1/nBfPyJEfGLvJ6vRERbPr49H96UT19XiToL6m2OiF9HxHdqtc6I2BwRv4mIWyLipnxcTb3v+WMvjogrIuLOiLgjIp5ca3VGxOn56zj2tz8i3lFrdeaP/cf5Z2hDRHwp/2zV4vr59rzGjRHxjnxc1V/PmKFte0S8Jp//7oh4TbnqnRVSSv6N+wOeBpwNbCgY9xHgPfnt9wAfzm8/D/guEMCTgF9UsM7jgLPz2wuA3wJn1lqt+ePNz2+3Ar/IH/+rwMX5+E8C/19++83AJ/PbFwNfqfD7/07gi8B38uGaqxPYDCwfN66m3vf8sT8L/EF+uw1YXIt1FtTbDOwA1tZancAJwH3AnIL18rW1tn4CZwEbgLlAC3AdcEotvJ7MwLYdWArcm/9fkt9eUul1tV7+ql5Arf4B68atiHcBx+W3jwPuym9/CrhkovmqUPO3gGfVcq35hudXwDlkJ7lpycc/Gbgmv30N8OT8dks+X1SovlXA94FnAt/JNzC1WOdmHh4yaup9BxaRfSlGLdc5rrZnAzfUYp1kIWNL/uXWkq+fz6m19RN4GfDpguG/AP60Vl5PjnLbDlwCfKpg/GHz+Xf4n7tLindMSunB/PYO4Jj89tgHf8zWfFxF5U2hjyNrJai5WvNdELcAncC1wD3A3pTS8AS1HKozn74PWFaJOoF/ItsgjubDy2q0zgR8LyJujog35eNq7X0/EdgF/Ee+++nfI2JeDdZZ6GLgS/ntmqozpbQN+AfgAeBBsvXtZmpv/dwAPDUilkXEXLIWgdXU2OtZoNS6ql1vXTFkTEPK4mvNHJYTEfOBrwPvSCntL5xWK7WmlEZSSo8layl4InBGlUt6mIh4PtCZUrq52rUU4SkppbOB5wJviYinFU6skfe9haxp+hMppccBPWTN0YfUSJ0A5H0ZXgh8bfy0Wqgz7yvwIrLwdjwwD7iomjVNJKV0B/Bh4HvA1cAtwMi4ear+ek6kVuuqZ4aM4u2MiOMA8v+d+fhtZCl9zKp8XEVERCtZwPhCSum/arlWgJTSXuAHZM26iyOiZYJaDtWZT18EdFWgvPOAF0bEZuDLZLtMPlaDdY79qiWl1Al8gyy41dr7vhXYmlL6RT58BVnoqLU6xzwX+FVKaWc+XGt1Xgjcl1LalVIaAv6LbJ2txfXz0ymlx6eUngZ0k/UXq7XXc0ypdVW73rpiyCjelcBYL+LXkPV/GBv/6rwn8pOAfQVNb2UVEQF8GrgjpfTRWq01IlZExOL89hyyfiN3kIWNl05S51j9LwWuz39hlFVK6c9SSqtSSuvIms2vTyn9fq3VGRHzImLB2G2yfgQbqLH3PaW0A9gSEafnoy4Abq+1OgtcwkO7SsbqqaU6HwCeFBFz88/+2OtZU+snQESszP+vAV5M1pG61l7PMaXWdQ3w7IhYkrcuPTsfp4lUu1NILf6RbWgeBIbIfo29gWxf5veBu8l6Sy/N5w3gUrI+Br8B1lewzqeQNe3dRtYkeQvZ/s+aqhV4NPDrvM4NwPvz8ScBvwQ2kTVRt+fjO/LhTfn0k6qwDpzPQ0eX1FSdeT235n8bgffm42vqfc8f+7HATfl7/02y3vi1WOc8sl/5iwrG1WKdHwTuzD9Hnwfaa239zB/7f8gC0K3ABbXyejJD23bg9fnrugl4XaVe13r884yfkiSpLNxdIkmSysKQIUmSysKQc4UoEQAABuNJREFUIUmSysKQIUmSysKQIUmSysKQoboSEQfLvPx35KdCPurHi+wqmNdFdmXPl89MhQ97jMPqLfG+iyPizTNUxwci4k9mYlmTLP+xEfG8mXq8iHhZZFeF/cHMVFj04x72mkfE8RFxRSVrkCrJkCEd7h1kF3GbCY8DSCk9NqX0lRla5nhHU+9isit1Tikizo+Iy6f5GDPlsWTngJkpbwDemFJ6xgwusxiHveYppe0ppZdOMb9U1wwZqnv/f3vnGmJVFcXx339EnZoxohQrrSQ1xMKKEiNFpyC/9EZ7YdH0gPqSQe8oTItK7VMPgrJMIpPS0qCkB5mPctIyH42mGD5CInqAZR+yrNWHtW5zul7vHaWbDq0fbNxnnf1Ye+8z96yz93YvSQMlvRPOwpZJGhLyWZKelLRc0hZJ40PeIOkZSRslvS9poaTxkibiPiE+LH7hSnpE0lpJn0jqW6H+oyQtkLQu0gyLEw9fBobHTMbAsjyDYpZjraTPow2S9LikdklflGY/4iW/WNK80Hl2pN1LX0ljJbVFmXMlNUs6UdJmSb2j7cskjQWmAgNDv8frNDZ3Sfo0+mZKyAbELMIMSeslvSc/CRZJwyPtmkJf9AAeAq4smxUaGv2yJfqiUv1XR1+2S5oWskn4QXYvlLc7+vVpSZtifBYWnpttknpH/CxJiyPeJGmmpJVyR3CXhPyUkK2JNg2mrM+jL9ojfaOkF0Pf1ZLODXmrpDfiGd8safq/NT5JUncO9mlgGTLsTwB+qSD7ABgc8RH48ckAs/ATDxuAocBXIR8PLAz5MbhvhfFxbxsFF+r4iaoXRXw68ECF+p8CHoz4ecCaiLcQp4ZWyLMCuCzijfhsxDjcQ2033BPk17hr6Rbcg2b/0LkNd5D2D32B3sBSoCmu76HjdNWboi/uItxUU+byukqftwCzaqSZDNxZJhsLPIefnNiAuyYfHfXuAU6PdK8B10S8nQ735FNL+gGtwNNl9S3HT7zsjZ/W2b2s/uOiD/vgztoWAZfGvcVUOFkSPwK7NAbHATsrPRvAWcDiiD9a0P9I3E9HUzwXE0LeAzisvM+L18AdwMyIDwndG6PtW3DfI43AduD4g/23mCFDZ0LJqU6SdEnkHmjPAeZKKol7FpIsMLM/gQ2FWYhRwNyQf6vq6/K/4S9HcLfa51dIMwo3EDCzRXIX10dU0bkX0M/M5keeX0M+CphjZn/gTpuWAMOBn4GVZrYj0q3BX04flRV9Nm5MfRx90QM3SDCz5yVdDtyCLz3URNIKvC+bgaOiXoB7zKwzvhrGRlgd183AYPzludXMSuWtAgbI/dv0MrO2kL8CXFil/LfNbDewW9J3uGG2o3B/OG4IfB/tmY0bOQuqlDmajjH4RtKiTrbzYnXsEWkETsD7/n5J/YE3zGxz4RmtxCjcMMHMNkraDpwc9z4ws5+iHRuAE/mnu/EkOSRJIyPp6jQAO83dyFdidyFe9Rd+H/xuZqWz9//g4P3NFNuxLz0EvG9mV+91wzeH9o/LZmBXrQrNbETkbQFazax1/1RGwGNm9myZLgPYuz2H7WfZVCij3mOzh44l5saCXMA4M9tUlv7LMNQuABZKuhmfkTgQ/uu2Jsm/Qu7JSLo0ZvYzsDW+0ktr6qfVyPYxMC72J/TFlwNK7AJ67acay4AJUX8L8EPotS+ddwE7JF0aeXqGEbAM33fQTVIf/Kt6ZY26i/p+AoyUNCjKbZJU+hKeBswGJgEzKuStB+8CN8RsE5L6KbxzVsLMdgK7JI0I0VWF2wei60pgTOxF6YZ7WV1SI89SOsbgWKC4MXQbcGbExxXk7wK3KqYpJJ0R/54EbDGzJ3HPnsNqtKP4HJ2Mz4aUGy5J0qVIIyPpahwuaUch3I7/MN8oqeSV9JIaZbyOT6tvwDdnfo7veQDfQ/BOjSWUciYDZ0pah+8juK56cgCuBSZGnuX43pD5uLfStfj+gbvN3aVX4299Y1mgFZgT5bYBQySNwZcOppnZbOA3Sdeb2Y/40kp7+QbIA+SB4tiY2Xv4kkebpC+AedQ2FG4EZsTSTBMd4/IhvtGz0/8d2Nwt972Rdy2wyszerJ6L+bg3zg3AS8RyUzAFeELSZ/hsQomHge7AOknr4xrgCqA92nIq8FKNPn8GaIi+ehWfPdpNknRh0gtr8r9EUrOZ/SLpaPyLd2QnXuhJnSmNS8TvBY41s9sOoj6z8M27eZZFkhwAua6X/F95KzYa9gAeTgPjkOECSffhv03b8ZmZJEm6KDmTkSRJkiRJXcg9GUmSJEmS1IU0MpIkSZIkqQtpZCRJkiRJUhfSyEiSJEmSpC6kkZEkSZIkSV1IIyNJkiRJkrrwF8fTzu4KSESRAAAAAElFTkSuQmCC\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "dist = {}\n",
        "for _, row in train_df.iterrows():\n",
        "  length = len(row['story'].split()) + len(row['question'].split())\n",
        "  dist[length] = dist.get(length, 0) + 1\n",
        "\n",
        "sorted_keys = sorted(dist.keys())\n",
        "total = sum(dist.values())\n",
        "dist = {k: sum([dist[i] for i in sorted_keys if i < k])/total for k in sorted_keys}\n",
        "plt.plot(list(dist.keys()), list(dist.values()))\n",
        "plt.xticks([i for i in range(100, max(dist.keys()), 100)])\n",
        "plt.xlabel('Length of context + Length of question')\n",
        "plt.ylabel('Dataset percentage')\n",
        "plt.title('Percentage of the samples with length of context + question less than a certain value.');"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "zkr6zN-UBKqn"
      },
      "source": [
        "As we can see, more than 80% of the dataset has an input length less or equal than 400, this will be very useful when defining the input length of our models. In fact, while a longer input means less truncated data, it also means harder computation and more RAM required to run our models.\n",
        "\n",
        "Let's check how many samples are under 400 length:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 9,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "rG9yl0YgWEmF",
        "outputId": "ea90243f-e889-4ed0-d4e1-a409efd35593"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9765217188665175"
            ]
          },
          "metadata": {},
          "execution_count": 9
        }
      ],
      "source": [
        "dist[400]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uZLiGtbSXABp"
      },
      "source": [
        "Very well most samples are below the 400 threshold. Let's set this value as our input length for now."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 10,
      "metadata": {
        "id": "7zHUWjr-W7t-"
      },
      "outputs": [],
      "source": [
        "input_len = 400"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "NM64LOuGXH1U"
      },
      "source": [
        "Let's now check the same thing but for the length of the answer"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "id": "ya8YwSHhDQX9",
        "outputId": "330ab1d7-bfce-4a3b-ee8b-0839d6066900"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAHwCAYAAAC2blbYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3debhkVXnv8e97eqAZmsluJ5pRcUDjlBY0ON3rBJhIYhwg0ThFzE1INGqiJgaVxCTGRKMJDmi4zqDiNRKDYxxwAkFFwiCxRbEbGZqxG3o4fU6994+9qrsozlCn6ao669T38zznOVV779q1du1d9au19qq1IzORJEn1GRt2ASRJ0s4xxCVJqpQhLklSpQxxSZIqZYhLklQpQ1ySpEoZ4iIifisi1kbE7RHxyB6Wf1JErBtE2QYhIg6JiIyIxUN47sdHxJUzzB9a2YahbOv9h/C8czqmI+LrEfH7/SzTLM//wYj4m2E9/3wUEX8RER8YwvMO5ZhtWzAhHhE/j4jNJYiuLwf5XsMuV1tEvCkiPjrsckzjH4GTM3OvzPxh98xhH6QLWWZ+MzMf2L5fjuOnDLNMo6CmYzoiXhQR3xp2OQZtrl+UMvNvM3NoX6yGZcGEePEbmbkX8ChgNfCGuTw4GgvtNenFwcBlwy6ERseotCxo7kb4c3jnZOaC+AN+Djyl4/7bgM+V248BvgPcCvwIeFLHcl8H3gJ8G9gM3B94CPBl4GbgeuAvyrJjwOuAnwI3AZ8E9i/zDgESeCHwC+BG4C/LvGOAcWAbcDvwozL9xcAVwEbgKuDlXdv058C1wC+B3y/rv3+ZtxtNDfoXpYzvBXaf5rUZo/lCczVwA/BhYJ+yjtvLeu8AfjrFY8/rmH878DzgScA64NVlfdcCL+54zFzKdn/gG8Bt5TX7RMe8dwJrgQ3A94HHd8x7E/Ap4KPl9ftv4AHA60uZ1gJP69rPfwd8r6zvs1Psu8Xl/j7Av5Xtugb4G2DRbOXt2q4PAa8utw8o6/+jcv9+NMfWWPu1LNM/ArRojsPby/5vl+0ux9U0z/sM4IdlG9cCb+qYN+O6gCOBi8pjrwfePpdtKfd/HbiY5r32HeBhXe/R1wKXAFvbr3dX+Xs6xpn9GLwH8B9lWy4s+/BbO3tMT1HOrwO/33H/JTTv5VuALwIHl+kBvKOscwPNcfrQMu844HKa4/ca4DVTPM+DgS3AZCnrrWX6B4HTgP8sj78AuN8c3jufpPkc2EjzBX71DNs67bqmWHZ34J9oPmtuA77Vsc/m8jn8sbLNW8p2/2uP2/XRXo71rjIfBVxHeY+Xab8FXNLxvvhuKfe1wL8CS6c5ZruPixdRjrty/0HsyJYrgedO91r2+jf08N1Vf3SEOHBgOTD/muZD56byhhkDnlrur+x40X9BE9yLgeVlR70aWFbuH1WWfQVwPrCK5gPmfcCZXQfN+8uB/HCaD6oHdx9gHWV+Bs2HYABPBDYBjyrzjikH1kOAPWjCqvNgeQdwDrB/KeN/AH83zWvzEmANcBiwF/D/gI9MdRBO8/g7zaf5wJsATgWWlNd2E7DfTpTtTOAvy75ZBjyuY97zaT6MF5f9cR2wrOP13AI8vcz/MPCzsq4lwMuAn3V9SFwDPBTYE/g0d33Dt0P8M2Xf7gnckyb4Xz5bead4zf+j3P4dmi9+n+iY99mO13LdVMdxL8fVFM/7JOBXSvkeRhN+v9njMfpd4AXl9l7AY+a4LY+kCaujgEU0H6A/B3br2LaLad6f032p6+kYZ/Zj8KzytwdwBM0H/7emep5e1jdFOb9O+bAGjqd5fz2Y5lh8A/CdMu/pNGGzL837/MHAfcq8aykhBOxHee9P8Vwv6ix7mfZBms+xI8tzfgw4a47vnePKfvo74PwZ3v/TrmuKZU8rr80BZd2/RvNZOdfP4SV0BWKP29X9nu71ffNT4Kkd9z8FvK7c/lWaLyCLy3qvAF45zTF7pzJ37juaz5O1NJW3xTTvlxuBI6Z77Xv5G3r47qo/mg+I22m+LV0NvLvsvNfSEVhl2S8CL+x40U/tmHci8MNpnuMK4Mkd9+9DU7tu79wEVnXM/x5wQvcBNsM2/DvwinL7DDqCj6YGmOV/0NQiOr95P5aO0Opa738Bf9hx/4HtcncfhNM8fqoPvM101KRoPrwfsxNl+zBweufrNkM5bgEe3vF6frlj3m+U/d+uMS8v5d63Yz//fcfyR9C0jizq2HeLgXvRvNl371j2ROBrcykvzZezW2g+sN4LvJwdNe4PAa/qeC17CfEpj6seXrN/Bt7Ry7poaqhvBlbs5La8B/jrrsdeCTyxY9teMkt5ezrGZzkGF9Ec3w/smLe9Jj7XY3qacn6dHSH+eeClHfPGaL4AHAz8b+B/SrnGutbxi/Ja7j3La/Iipg7xD3TcPw748RzeO1/pei9s7uV46l5X1/Sx8hpONW9On8Pdr/Ectqs7xHt635Tj44xye3k59g6eZtlXAp+Z6ljqLjN3DvHnAd/sWtf7gDf2+tpP9bfQzjv8Zmbum5kHZ+YfZuZmmjfScyLi1vYf8DiaAG5b23H7QJpvZVM5GPhMx3quoGnyuVfHMtd13N5EU6OZUkQcGxHnR8TNZX3HASvK7Pt2lavz9kqaGsb3O8ryhTJ9Kvel+WLTdjU7Amtn3ZSZEx3329s617L9Oc0H9vci4rKIeEl7RkS8JiKuiIjbynr2YcfrA00ts20zcGNmTnbchzu//p2v4dU03/Y71wfNPl4CXNtR/vfR1MhnLG+nzPwpzQfBI4DHA58DfhkRD6RpdfnG1C/HtHo6riLiqIj4WkSsj4jbgD+YYhunW9dLaU5J/DgiLoyIX5/jthwMvLrrvXYgzfHX1rkPZtLLcTTTMbiY6d8/05lufbM5GHhnRzlvpjlGDsjMr9I0v54G3BARp0fE3uVxv03znr86Ir4REY/t4bk6TXtM9PDe6X7ssun6KfSwrrYVNK1TU31+zvVzeEpzKEtbr5/HHweeFRG7Ac8CfpCZV5fnfEBEfC4irouIDcDfzvKc0zkYOKrrNfhd4N47sa7tRqFzyVqab4Avm2GZ7Fr+hBnW9ZLM/Hb3jIg4ZJZydD4H5WD5NPB7NM2R2yLi32ne/NA0ta3qeMiBHbdvpAmph2TmNbM8LzTn1A/uuH8QTdPh9VMvfrfMqWyZeR1N0zcR8TjgKxFxHs2b+8+BJwOXZWYrIm5hx+uzMzpfw4Noams3dk1fS1MTX9H1gT5jeTNzzRTP9w3g2TTnz66JiG/QNDHvR9OsPJWcZnqvPk4TGsdm5paI+Gd6/MDJzJ8AJ5ZORc8Czo6Ie2TmHT1uy1rgLZn5lpmepsftmOsx3mk9zfG9iqYWDHfex7tae7s/NtXMzHwX8K6IuCfNueg/A/4qMy8Ejo+IJcDJZd5U5ZzTMRERj2cXvXfmuK4baZrp70dzzrvTXD+H73J/V27XXZ448/KIuBo4luaU0cc7Zr+Hpp/JiZm5MSJeSfNemModNF8+2zoDei3wjcx86t0tb6eFVhOfykeB34iIp0fEoohYFs1vQldNs/zngPtExCsjYreIWB4RR5V57wXeEhEHA0TEyog4vsdyXA8c0tHrcinNuaL1wEREHAs8rWP5TwIvjogHR8QewF+1Z2Rmi+ZczzvKBwMRcUBEPH2a5z4T+NOIODSan939Lc05zbuE1AxlP6yXBedatoh4Tse+uIXmjduiadKaoHl9FkfEKcDeU61jDp4fEUeU1/NU4OyOmnu7/NcCXwL+KSL2joixiLhfRDxxlvJO5Rs0H87nlftfL/e/1f28HXp+raexHLi5BPiRNB9IPYmI50fEyrIPby2T29vWy7a8H/iD0hoQEbFnRDwjIpbPdSN24hjvfOwkTb+PN0XEHhHxIJovy53u7uvc6b3A6yPiIaWc+0TEc8rtR5fXYwnNB/wWoBURSyPidyNin8zcRtNRa7rj6HpgVUQs7bE8u/K90/O6yj47A3h7RNy3fN4+tlRY5vo5DHfdR/34TOj0cZp+T0+gOSfe+bwbgNvLsfR/ZljHxTQ1+j2i+QnjSzvmfQ54QES8ICKWlL9HR8SD706hF3yIZ+Zamo4nf0Gz89fSfBOectszcyNNp4vfoGmK+Qnwv8rsd9J0tPlSRGyk6eR21FTrmUL7oLgpIn5QnudPaML6FpoP23M6yvF54F3A12g6zZxfZm0t/1/bnl6aeL5Cc657KmfQ9Hw+j6bz1xbgj3ssNzTnmj5UmoCe28Pycynbo4ELIuJ2mu1/RWZeRXO+7As0NamrS5l7bYqdzkdoziVeR9Ps9yfTLPd7NF+yLqfZN2ezo9lvuvJO5Rs0HwDt4PsWzbf086ZZHppORm8or/Vretimbn8InFqOz1Nojq9eHQNcVrbtnTTnD9unJWbdlsy8iKaV4l9pXrc1NOcEd9ZcjqNuJ9M0tV5Hs9/PZMd7B+Z+TE8rMz8DvBU4q5TzUpoaHTQh836a1+Nqms5cbyvzXgD8vDzmD2iaVqfyVZqOutdFxI09FGlXvnfmuq7X0PTAv5DmtMJbafoCzOlzuHgn8OyIuCUi3rWLt2sqZ9KcHvpqZna+zq+h+XzeSLMvPzHDOt5B09fmepr+IttbZ8pn/tNoWnp/SXNsvpWmMtcerObzcy10lJPrmufKt7VLaXr69lqDVhERX6fp9DLwEZ00fBHxVuDemfnCYZdF2pUWfE28ZtEMh7pbROxH843tPwxwaXYR8aCIeFhp1j+SplnzM8Mul7SrGeLz28tpfubyU5pe8DOdi5G0w3Ka8+J30DR//hPNAD/SgmJzuiRJlbImLklSpQxxSZIqVd1gLytWrMhDDjlk2MWQJGkgvv/979+YmVOOelldiB9yyCFcdNFFwy6GJEkDUUaTm5LN6ZIkVcoQlySpUoa4JEmVMsQlSaqUIS5JUqUMcUmSKmWIS5JUKUNckqRKGeKSJFXKEJckqVKGuCRJlTLEJUmqlCEuSVKlDHFJkirVtxCPiDMi4oaIuHSa+RER74qINRFxSUQ8ql9lkSRpIepnTfyDwDEzzD8WOLz8nQS8p49lkSRpwVncrxVn5nkRccgMixwPfDgzEzg/IvaNiPtk5rX9KtNClJlkltvl/o7bkJT7Odt6ppnOLA/sYX25fVre6X6/zbbNktQPEbD3siUDea6+hXgPDgDWdtxfV6ZVH+Jbtk1y1fo7WHfLJq7fuJUbNmzh5jvGuX3rBHdsnWDT+CRbtk2yYcsEm8cn2TbZYqKVTLaSViuZaCWtEs5J0sq7BqABJUnz097LFnPJm54+kOcaZoj3LCJOomly56CDDhpyae6q1Uou+NnNnPeT9XzzJ+u57Jcb7hSyYwH77rGU5csWs+fSxey+dBG7L13EvfdZxrIli1i6aIzFi4LFY2NEwOKxYCyCiCCiefxYxPb1bb8VQTT/iDK1uc2O252Pi/bjd0zrFFNPvvNzzkHn+jrLN0gx6CeUNPKWLh5cn/Fhhvg1wIEd91eVaXeRmacDpwOsXr163tRBM5PPX3od//jFK7nqxjtYPBY88qB9+eP/dX8ecO/lHLjfHtxnn2XcY6/dWDRmmEiSdq1hhvg5wMkRcRZwFHBbTefDb7p9K6/99CV85YobeOC9lvPPz3sETz3iXuy5WxWNG5KkBaBviRMRZwJPAlZExDrgjcASgMx8L3AucBywBtgEvLhfZdnV1t68iRf82wX88rYtvOEZD+bFRx9qTVuSNHD97J1+4izzE/ijfj1/v9y6aZwXnvE9btm0jTNfdhS/evD+wy6SJGlEOWLbHP352Zew7pbNfOCFqw1wSdJQGeJz8J01N/Kly6/nlU89nEcfYoBLkobLEJ+Dt37hxxyw7+685OhDh10USZIM8V5des1t/Gjdbbz8iYexbMmiYRdHkiRDvFefuHAtSxePcfzDDxh2USRJAgzxnmydmOSzF1/DsQ+9N/vsMZjxcCVJmo0h3oMLrrqZDVsmeObD7zvsokiStJ0h3oOvXHE9y5aMcfT9Vwy7KJIkbWeIzyIz+crl1/P4w1faoU2SNK8Y4rNYc8Pt/PK2LTz5QfccdlEkSboTQ3wWF6+9FYDVDu4iSZpnDPFZXLLuNvbabTGHrdhz2EWRJOlODPFZXLLuVh56wN6MeZUySdI8Y4jPYHyixRXXbuThq/YddlEkSboLQ3wGV163kfHJFr+yap9hF0WSpLswxGdw5fUbAXjwffYeckkkSborQ3wGv7jpDsYCDtxvj2EXRZKkuzDEZ3D1zZu47767s3SxL5Mkaf4xnWZw9U2bOPge1sIlSfOTIT6DX9y8iYP29/fhkqT5yRCfxsYt27j5jnFr4pKkecsQn8bVN20C4OD9DXFJ0vxkiE/jFzc3IX6QNXFJ0jxliE9je4hbE5ckzVOG+DSu37CFvXZbzPJlS4ZdFEmSpmSIT+OGjVu55/Ldhl0MSZKmZYhPY/3GrawwxCVJ85ghPo311sQlSfOcIT6NGzZs4Z7Llw27GJIkTcsQn8IdWye4Y3ySldbEJUnzmCE+hfUbtwLYnC5JmtcM8Snc0A7xvQ1xSdL8ZYhP4YaNWwBsTpckzWuG+BR2NKfbsU2SNH8Z4lO4YeNWFo8F++7uaG2SpPnLEJ/CjRu3smKv3Rgbi2EXRZKkaRniU7ht8zb23cNauCRpfjPEp7Bhyzb29sInkqR5zhCfwobNE+y9++JhF0OSpBkZ4lOwJi5JqoEhPoUNm7extz3TJUnznCHepdVKNm6dYO9lNqdLkuY3Q7zL7eMTZGJNXJI07xniXTZs3gbAcmvikqR5zhDvsnHLBIAd2yRJ854h3qVdE7c5XZI03xniXTZYE5ckVcIQ77KjJu45cUnS/GaId9mwpYS4NXFJ0jxniHfZsLlpTrd3uiRpvjPEu2zYso09ly5i8SJfGknS/GZSdXHIVUlSLQzxLl78RJJUC0O8i5chlSTVwhDvcsf4BHssNcQlSfOfId5l8/gkeyxdNOxiSJI0K0O8y+Ztk+xuiEuSKmCId9mybZLdlxjikqT5zxDvsmncEJck1cEQ75CZNqdLkqphiHfYOtEiE5ZZE5ckVcAQ77Bl2ySAzemSpCoY4h02t0Pc5nRJUgUM8Q6bx62JS5LqYYh3sCYuSaqJId7Bc+KSpJoY4h02jVsTlyTVwxDv4DlxSVJNDPEO7XPi/k5cklQDQ7zDFju2SZIq0tcQj4hjIuLKiFgTEa+bYv5BEfG1iPhhRFwSEcf1szyzsTldklSTvoV4RCwCTgOOBY4AToyII7oWewPwycx8JHAC8O5+lacXm7e1ALyeuCSpCv2siR8JrMnMqzJzHDgLOL5rmQT2Lrf3AX7Zx/LMqn1OfLfFnmWQJM1/i/u47gOAtR331wFHdS3zJuBLEfHHwJ7AU/pYnlm1ryUeEcMshiRJPRl2lfNE4IOZuQo4DvhIRNylTBFxUkRcFBEXrV+/vm+F2TQ+Yac2SVI1+hni1wAHdtxfVaZ1einwSYDM/C6wDFjRvaLMPD0zV2fm6pUrV/apuLB5vGWnNklSNfoZ4hcCh0fEoRGxlKbj2jldy/wCeDJARDyYJsT7V9WexZZtkyxbMuzGCUmSetO3xMrMCeBk4IvAFTS90C+LiFMj4pllsVcDL4uIHwFnAi/KzOxXmWazedukzemSpGr0s2MbmXkucG7XtFM6bl8OHN3PMszF5vFJm9MlSdWw7bhDUxPv6/caSZJ2GUO8Q/MTM18SSVIdTKwOm7fZnC5Jqoch3mHTuB3bJEn1MMQ7bN02yW6LDXFJUh0M8Q7jky2WOm66JKkSJlaHbZPJkkWOmy5JqoMhXky2kslWsmSRL4kkqQ4mVrFtsrmWuCEuSaqFiVW0Q3ypIS5JqoSJVWybbIZst2ObJKkWJlZhc7okqTYmVjE+0Q5xe6dLkupgiBfbz4nbnC5JqoSJVbTPiducLkmqhYlVeE5cklQbE6sYn/ScuCSpLoZ4sW3C34lLkupiYhXbz4nbsU2SVAkTqxifnAQ8Jy5JqoeJVYxPtHune05cklQHQ7xw7HRJUm1MrMKfmEmSamNiFdtD3I5tkqRKmFjFePsqZtbEJUmVMLEKfycuSaqNiVXsaE63d7okqQ6GeGHHNklSbUyson1OfPGYNXFJUh0M8WLbZIuli8aIMMQlSXUwxIttEy1Ha5MkVcUQL7ZNtvyNuCSpKqZWMT6ZdmqTJFXF1Cra58QlSaqFqVWMe05cklQZQ7zYNtmyOV2SVBVTqzDEJUm1MbWK8clkqb3TJUkVMbWKbRN2bJMk1cXUKprfiduxTZJUD0O88Jy4JKk2plbhYC+SpNqYWoWDvUiSamNqFU1zuufEJUn1MMSL5ipmvhySpHqYWsX4ZHoVM0lSVUytwnPikqTamFqF58QlSbUxxItxz4lLkipjagGtVjLR8nfikqS6mFrAtlYLwAugSJKqYmoB2yYTwI5tkqSqmFo0vxEHWGzHNklSRQxxYKLV1MQXWxOXJFXE1AImS4gvCmvikqR6GOLAZJaa+JghLkmqhyEOTJaObWOGuCSpIoY4MFF+YmZNXJJUE0McaJXm9EWGuCSpIoY4O3qnG+KSpJoY4nT0TjfEJUkVMcTxJ2aSpDoZ4nSEuCO2SZIqYohjTVySVCdDnB0h7k/MJEk16SnEI+JxEfHicntlRBza32INlh3bJEk1mjXEI+KNwGuB15dJS4CP9rNQg+ZPzCRJNeqlJv5bwDOBOwAy85fA8n4WatAmHexFklShXkJ8PDMTSICI2LO/RRq89tjphrgkqSa9hPgnI+J9wL4R8TLgK8D7+1uswbImLkmq0awhnpn/CJwNfBp4IHBKZv5LLyuPiGMi4sqIWBMRr5tmmedGxOURcVlEfHwuhd9V7NgmSarR4l4WyswvA1+ey4ojYhFwGvBUYB1wYUSck5mXdyxzOE2HuaMz85aIuOdcnmNX8SdmkqQa9dI7fWNEbOj6WxsRn4mIw2Z46JHAmsy8KjPHgbOA47uWeRlwWmbeApCZN+zshtwd7RAfc7AXSVJFeqmJ/zNNTfrjQAAnAPcDfgCcATxpmscdAKztuL8OOKprmQcARMS3gUXAmzLzCz2WfZeZ2F4Td+wbSVI9egnxZ2bmwzvunx4RF2fmayPiL3bB8x9O80VgFXBeRPxKZt7auVBEnAScBHDQQQfdzae8q5Zjp0uSKtRL1XNT6Xw2Vv6eC2wp83KGx10DHNhxf1WZ1mkdcE5mbsvMnwH/QxPqd5KZp2fm6sxcvXLlyh6KPDcTjp0uSapQLyH+u8ALgBuA68vt50fE7sDJMzzuQuDwiDg0IpbSNMOf07XMv1Oa4yNiBU3z+lVz2YBdwZ+YSZJqNGtzemZeBfzGNLO/NcPjJiLiZOCLNOe7z8jMyyLiVOCizDynzHtaRFwOTAJ/lpk3zXUj7q7JyRZgiEuS6jJriEfEMuClwEOAZe3pmfmS2R6bmecC53ZNO6XjdgKvKn9DUwZsM8QlSVXppTn9I8C9gacD36A5t72xn4UatMmWNXFJUn16CfH7Z+ZfAXdk5oeAZ3DXn4pVrbSmO9iLJKkqvYT4tvL/1oh4KLAPMJSR1frFmrgkqUa9/E789IjYD3gDTe/yvYC/6mupBsyfmEmSatRLiP9XGRb1POAwgIg4tK+lGrBWK4mAMWvikqSK9NKc/ukppp29qwsyTBOttBYuSarOtDXxiHgQzc/K9omIZ3XM2puOn5otBJOZng+XJFVnpub0BwK/DuzLnQd72Uhz9bEFY3LSEJck1WfaEM/MzwKfjYjHZuZ3B1imgbMmLkmqUS8d29aUq5Ud0rl8LyO21WKyZYhLkurTS4h/Fvgm8BWa8c0XnIlWOtCLJKk6vYT4Hpn52r6XZIha1sQlSRXq5Sdmn4uI4/pekiHyJ2aSpBr1EuKvoAnyLRGxISI2RsSGfhdskFqtZNEiQ1ySVJderie+fBAFGSZr4pKkGs1aE4/G8yPir8r9AyPiyP4XbXD8iZkkqUa9NKe/G3gs8Dvl/u3AaX0r0RA42IskqUa99E4/KjMfFRE/BMjMWyJiaZ/LNVBNTbyX7zOSJM0fPV1PPCIWAQkQESuBVl9LNWCT/k5cklShXkL8XcBngHtGxFuAbwF/29dSDdhEK70MqSSpOr30Tv9YRHwfeDIQwG9m5hV9L9kAtayJS5IqNGuIR8RjgMsy87Ryf++IOCozL+h76QZkotXyJ2aSpOr00pz+Hpoe6W23l2kLRquFvdMlSdXpJcQjM7N9JzNb9NarvRoTrZYhLkmqTi8hflVE/ElELCl/rwCu6nfBBmkyrYlLkurTS4j/AfBrwDXAOuAo4KR+FmrQJq2JS5IqNGOzePl9+Dsy84QBlWcoJhyxTZJUoRlr4pk5CRy80EZo69ZKf2ImSapPLx3UrgK+HRHnAHe0J2bm2/tWqgFzsBdJUo16CfGflr8xYEFeltTBXiRJNeplxLY3A0TEHpm5qf9FGjyvJy5JqlEv1xN/bERcDvy43H94RLy77yUboFbLjm2SpPr08hOzfwaeDtwEkJk/Ap7Qz0IN2oQhLkmqUE8X0c7MtV2TJvtQlqFppSEuSapPLx3b1kbErwEZEUuAVwAL6ipmE3ZskyRVqNcR2/4IOAD4JfCIcn/BmJz0J2aSpPr00jv9RuB3B1CWoZl0sBdJUoV66Z1+WET8R0Ssj4gbIuKzEXHYIAo3KA72IkmqUS/N6R8HPgncB7gv8CngzH4WatAc7EWSVKNeQnyPzPxIZk6Uv48Cy/pdsEFysBdJUo166Z3++Yh4HXAWkMDzgHMjYn+AzLy5j+Xru1YrAVg01tOv7SRJmjd6CfHnlv8v75p+Ak2oV31+fGJ7iA+5IJIkzVEvvdMPHURBhmXSmrgkqVIjn1yT2YS4HdskSbUxxCebEPcnZpKk2hji1sQlSZXqZbCX/+plWq0mWi3AmrgkqT7TdmyLiGXAHsCKiNgPaKfc3jTjqC8IJcOtiUuSqjNT7/SXA6+kGaXtBx3TNwD/2s9CDVK7Ju5gL5Kk2kwb4pn5TuCdEfHHmfkvAyzTQLVr4l5PXJJUm146tp0REW+IiNMBIuLwiPj1PpdrYNo18cWLDHFJUl16CiPha+QAABPsSURBVHFgHPi1cv8a4G/6VqIBaw/2MmZzuiSpMr2E+P0y8x+AbQCZuYkdndyq50/MJEm16iXExyNid5px0omI+wFb+1qqAZpwsBdJUqV6uQDKG4EvAAdGxMeAo4EX9bNQg9SyJi5JqlQvF0D5ckT8AHgMTTP6KzLzxr6XbEDaVzGzJi5Jqk0vI7YdDWzJzP8E9gX+IiIO7nvJBmT79cTt2CZJqkwv58TfA2yKiIcDrwJ+Cny4r6UaoJLh/k5cklSdXkJ8IjMTOB44LTNPA5b3t1iD0z4nbkVcklSbXjq2bYyI1wPPB54QEWPAkv4Wa3C2h/jC+dWcJGlE9FITfx7NT8pempnXAauAt/W1VINUmtNtTZck1aaX3unXAW/vuP8LFuA5cXunS5Jq00vv9MdExIURcXtEjEfEZETcNojCDcKO5nRJkurSS3P6vwInAj8Bdgd+H3h3Pws1SKUiTtizTZJUmV5CnMxcAyzKzMnM/L/AMf0t1uC0a+K2pkuSatNL7/RNEbEUuDgi/gG4lh7Dvwa5/SdmprgkqS69hPELynInA3cABwLP6mehBintnS5JqlQvIf6bmbklMzdk5psz81XAr/e7YIOyvXe6NXFJUmV6CfEXTjHtRbu4HEPTPicuSVJtpj0nHhEnAr8DHBoR53TMWg7c3O+CDUpaE5ckVWqmjm3foenEtgL4p47pG4FLell5RBwDvBNYBHwgM/9+muV+GzgbeHRmXtTLuneVdse2sQXTVU+SNCqmDfHMvBq4Gnjszqw4IhYBpwFPBdYBF0bEOZl5eddyy4FXABfszPPcXe1z4o6dLkmqzc6O2Lahh3UfCazJzKsycxw4i+ZKaN3+GngrsGVOJd9FEn8nLkmq086O2HZaD487AFjbcX9dmbZdRDwKODAz/7On0vbB9pq458QlSZUZ2oht5ZKmbwde3cOyJ0XERRFx0fr16+/uU99JOmKbJKlSvYT4nUZsi4g/7fFx19AMDNO2qkxrWw48FPh6RPwceAxwTkSs7l5RZp6emaszc/XKlSt7eOretRyxTZJUqZ0dse23e3jchcDhEXFo+RJwArD9p2qZeVtmrsjMQzLzEOB84JmD753e/LcmLkmqTS/XE786IlaW22/udcWZORERJwNfpPmJ2RmZeVlEnApclJnnzLyGwXDENklSrWYa7CWAN9LUwMfKpAngXzLz1F5WnpnnAud2TTtlmmWf1GOZdylHbJMk1Wqm5vQ/BY6mGYBl/8zcDzgKOLqcF18Y2jVx29MlSZWZKcRfAJyYmT9rT8jMq4DnA7/X74INitcTlyTVaqYQX5KZN3ZPzMz1wJL+FWmwHLFNklSrmUJ8fCfnVcUR2yRJtZqpd/rDpxleNYBlfSrPwDlimySpVjNdAGXRIAsyLLl9sJchF0SSpDka+Qtwej1xSVKtRj7E7Z0uSaqVIW7vdElSpUY+xLefEx/5V0KSVJuRjy7PiUuSajXyIb79UqRDLockSXNliFsTlyRVauRDvD1imxkuSaqNIW5NXJJUqZEP8VbLmrgkqU4jH+KlIm5NXJJUnZEPcUdskyTVyhD3KmaSpEqNfIiT6flwSVKVRj7EW+n5cElSnQzxTEdrkyRVaeRDPLEmLkmq08iHeMtz4pKkSo18iGc60IskqU6GeKbN6ZKkKo18iNs7XZJUK0Pc3umSpEqNfIh7TlySVCtDPJMxB06XJFVo5EPcc+KSpFoZ4p4TlyRVauRDPPEKZpKkOhnimV5LXJJUpZEP8VbL3umSpDqNfIgnjtgmSarTyIe4vdMlSbUyxDOHXQRJknbKyIc4CWO+CpKkCo18fLW8ipkkqVKGeOJgL5KkKo18iCd2bJMk1WnkQ7yV6e/EJUlVGvkQz0yHXZUkVckQTxx2VZJUpZEPcXunS5JqZYg71oskqVIjH+JpTVySVClD3BHbJEmVGvn48py4JKlWhrgjtkmSKjXyIZ7g78QlSVUyxDP9nbgkqUojH+ItR2yTJFVq5EPcEdskSbUa+RC3Ji5JqpUhbu90SVKlRj7ESa8nLkmq08iHeCvTEdskSVUa+fhqZRI2qEuSKjTyId4M9jLsUkiSNHcjH+Itz4lLkio18iGemdbEJUlVMsStiUuSKjXyId5y7HRJUqUM8fQqZpKkOo18iGemPzCTJFXJEPecuCSpUiMf4o7YJkmq1cjHlyO2SZJq1dcQj4hjIuLKiFgTEa+bYv6rIuLyiLgkIv4rIg7uZ3mm4ohtkqRa9S3EI2IRcBpwLHAEcGJEHNG12A+B1Zn5MOBs4B/6VZ7peE5cklSrftbEjwTWZOZVmTkOnAUc37lAZn4tMzeVu+cDq/pYnim1HLFNklSpfob4AcDajvvryrTpvBT4fB/LMyVr4pKkWi0edgEAIuL5wGrgidPMPwk4CeCggw7apc9tTVySVKt+1sSvAQ7suL+qTLuTiHgK8JfAMzNz61QryszTM3N1Zq5euXLlLi1kJvZOlyRVqZ8hfiFweEQcGhFLgROAczoXiIhHAu+jCfAb+liWaaVjp0uSKtW3EM/MCeBk4IvAFcAnM/OyiDg1Ip5ZFnsbsBfwqYi4OCLOmWZ1feP1xCVJterrOfHMPBc4t2vaKR23n9LP5++F58QlSbUa+RHbmsFeTHFJUn0Mcc+JS5IqNfIh3lxPfNilkCRp7gzxTDu2SZKqNPIh7ohtkqRajXyI2ztdklSrkQ9xR2yTJNXKELd3uiSpUiMf4q2EMVNcklQhQzzTxnRJUpVGPsQdsU2SVCtD3HPikqRKjXyIO2KbJKlWIx/i6YhtkqRKjXyINzVxQ1ySVJ+RDvHMBLB3uiSpSiMe4s1/m9MlSTUa6RBvlRS3d7okqUYjHuLNfyvikqQajXSIJ+WcuCkuSarQaIe458QlSRUb6RD3nLgkqWYjHuLNfyvikqQajXSI5/aauCkuSarPSIf4jpq4IS5Jqs9Ih7gjtkmSajbiId78t2ObJKlGIx3i23unm+KSpAqNeIg3/41wSVKNRjrEHbFNklSz0Q5xR2yTJFVspEO8fU7cDJck1WikQ9ze6ZKkmo10iO+oiZvikqT6jHSIp73TJUkVM8SxY5skqU4jHeI7BnsZckEkSdoJIx1f28+J26AuSarQiId489/WdElSjUY6xMHriUuS6jXSId6yY5skqWIjHuKO2CZJqtdIh7gjtkmSajbSIe6IbZKkmo10iDtimySpZoY4dmyTJNVppEPcEdskSTUb6fhyxDZJUs1GOsRLa7o/MZMkVWm0QzwdsU2SVK+RDnHHTpck1WykQ9ze6ZKkmo10iDvsqiSpZoY49k6XJNVppEMcx06XJFVspEN8+6VITXFJUoVGPMTbPzEbckEkSdoJhjjgJVAkSTUa6RBvR7g1cUlSjUY7xB2xTZJUsZEO8Var+W+GS5JqNNIhvqM53RSXJNVnpEPcEdskSTUb6RBPR2yTJFVsxEO8+T820q+CJKlWIx1fLa9iJkmq2IiHeLs5XZKk+ox0iG8fr82auCSpQqMd4o6dLkmq2EiH+I6fmJnikqT69DXEI+KYiLgyItZExOummL9bRHyizL8gIg7pZ3m6pdcTlyRVrG8hHhGLgNOAY4EjgBMj4oiuxV4K3JKZ9wfeAby1X+WZir3TJUk162dN/EhgTWZelZnjwFnA8V3LHA98qNw+G3hyDLBte8elSCVJqk8/Q/wAYG3H/XVl2pTLZOYEcBtwjz6W6U62d2yzPV2SVKEqOrZFxEkRcVFEXLR+/fpdtt7999yNRxy4L0sXVfEySJJ0J4v7uO5rgAM77q8q06ZaZl1ELAb2AW7qXlFmng6cDrB69epd1gb+1CPuxVOPuNeuWp0kSQPVzyrohcDhEXFoRCwFTgDO6VrmHOCF5fazga9meqJakqRe9K0mnpkTEXEy8EVgEXBGZl4WEacCF2XmOcC/AR+JiDXAzTRBL0mSetDP5nQy81zg3K5pp3Tc3gI8p59lkCRpobJHlyRJlTLEJUmqlCEuSVKlDHFJkipliEuSVClDXJKkShnikiRVyhCXJKlShrgkSZUyxCVJqpQhLklSpQxxSZIqZYhLklQpQ1ySpEoZ4pIkVSoyc9hlmJOIWA9cPexyVGYFcOOwCzEAo7Cdo7CNMBrbOQrbCG7nrnBwZq6cakZ1Ia65i4iLMnP1sMvRb6OwnaOwjTAa2zkK2whuZ7/ZnC5JUqUMcUmSKmWIj4bTh12AARmF7RyFbYTR2M5R2EZwO/vKc+KSJFXKmrgkSZUyxBeYiPh5RPx3RFwcEReVaftHxJcj4ifl/37DLudcRcQZEXFDRFzaMW3K7YrGuyJiTURcEhGPGl7J52aa7XxTRFxT9unFEXFcx7zXl+28MiKePpxSz01EHBgRX4uIyyPisoh4RZm+oPbnDNu5YPZnRCyLiO9FxI/KNr65TD80Ii4o2/KJiFhapu9W7q8p8w8ZZvl7NcN2fjAiftaxLx9Rpg/umM1M/xbQH/BzYEXXtH8AXlduvw5467DLuRPb9QTgUcCls20XcBzweSCAxwAXDLv8d3M73wS8ZopljwB+BOwGHAr8FFg07G3oYRvvAzyq3F4O/E/ZlgW1P2fYzgWzP8s+2avcXgJcUPbRJ4ETyvT3Av+n3P5D4L3l9gnAJ4a9DXdzOz8IPHuK5Qd2zFoTHw3HAx8qtz8E/OYQy7JTMvM84OauydNt1/HAh7NxPrBvRNxnMCW9e6bZzukcD5yVmVsz82fAGuDIvhVuF8nMazPzB+X2RuAK4AAW2P6cYTunU93+LPvk9nJ3SflL4H8DZ5fp3fuyvY/PBp4cETGg4u60GbZzOgM7Zg3xhSeBL0XE9yPipDLtXpl5bbl9HXCv4RRtl5tuuw4A1nYst46ZPzxrcHJpljuj43RI9dtZmlMfSVOzWbD7s2s7YQHtz4hYFBEXAzcAX6ZpQbg1MyfKIp3bsX0by/zbgHsMtsQ7p3s7M7O9L99S9uU7ImK3Mm1g+9IQX3gel5mPAo4F/igintA5M5u2ngX3k4SFul3Fe4D7AY8ArgX+abjF2TUiYi/g08ArM3ND57yFtD+n2M4FtT8zczIzHwGsomk5eNCQi9QX3dsZEQ8FXk+zvY8G9gdeO+hyGeILTGZeU/7fAHyG5k11fbspp/y/YXgl3KWm265rgAM7lltVplUpM68vHyAt4P3saGKtdjsjYglNsH0sM/9fmbzg9udU27kQ9ydAZt4KfA14LE3z8eIyq3M7tm9jmb8PcNOAi3q3dGznMeWUSWbmVuD/MoR9aYgvIBGxZ0Qsb98GngZcCpwDvLAs9kLgs8Mp4S433XadA/xe6SH6GOC2jmba6nSdS/stmn0KzXaeUHr8HgocDnxv0OWbq3IO9N+AKzLz7R2zFtT+nG47F9L+jIiVEbFvub078FSac/9fA55dFuvel+19/Gzgq6XVZV6bZjt/3PGlM2jO+3fuy8Ecs/3qMeff4P+Aw2h6t/4IuAz4yzL9HsB/AT8BvgLsP+yy7sS2nUnT9LiN5vzSS6fbLpoeoafRnJv7b2D1sMt/N7fzI2U7LikfDvfpWP4vy3ZeCRw77PL3uI2Po2kqvwS4uPwdt9D25wzbuWD2J/Aw4IdlWy4FTinTD6P5ArIG+BSwW5m+rNxfU+YfNuxtuJvb+dWyLy8FPsqOHuwDO2YdsU2SpErZnC5JUqUMcUmSKmWIS5JUKUNckqRKGeKSJFXKEJe00yLi6xGxehevc9+I+MOO+0+KiM/tyueQFgpDXFJPysAVg/jM2JfmaleSZmGIS/NcRPx7uaDNZR0XtSEibo+It5RrHJ8fEfcq058TEZeW6eeVaf8ZEQ8rt38YEaeU26dGxMvK7T+LiAvLxRza10s+JJprW3+YZkCLA5lGRDwtIr4bET+IiE+VMcPb17h/c5n+3xHxoDJ9ZTTXDb8sIj4QEVdHxArg74H7RXN95reV1e8VEWdHxI8j4mNlhCxp5Bni0vz3ksz8VWA18CcR0b7q057A+Zn5cOA84GVl+inA08v0Z5Zp3wQeHxH7ABPA0WX644HzIuJpNMN8HklzYY5f7bh4zuHAuzPzIZl59VQFLOH7BuAp2VyA5yLgVR2L3Fimvwd4TZn2RpphNx9Cc1nKg8r01wE/zcxHZOaflWmPBF5Jc83twzrKL400Q1ya//4kIn4EnE9TEz68TB8H2ueKvw8cUm5/G/hgqWEvKtO+CTyBJvz+k6ZmuwdwaGZeSTPO/tNohpb8Ac2VmdrPc3U210SeyWNoAvbb5XKNLwQO7pjfvshJZzkfB5wFkJlfAG6ZYf3fy8x12Vw05OKOdUgjbfHsi0galoh4EvAU4LGZuSkivk4z/jTAttwxbvIk5f2cmX8QEUcBzwC+HxG/ClxIU5O/iuaazytoau7fbz8V8HeZ+b6u5z8EuKOXotJcY/nEaeZv7S7nHG3tuL2z65AWHGvi0vy2D3BLCfAH0dR4ZxQR98vMCzLzFGA9cGBmjgNrgecA36Wpmb+Gphke4IvASzrOYx8QEfecQznPB46OiPuXx+8ZEQ+Y5THfBp5bln8asF+ZvhFYPofnlkaWIS7Nb18AFkfEFTQdvmZr1gZ4W+lAdinwHZqr2kET3Ddk5uZye1X5T2Z+Cfg48N2I+G+ac9Q9B2lmrgdeBJwZEZfQfFF40CwPezPwtFLO5wDXARsz8yaaZvlLOzq2SZqCVzGTNBQRsRswmZkTEfFY4D2Z+Yhhl0uqieeVJA3LQcAny2/Px9nRu15Sj6yJS5JUKc+JS5JUKUNckqRKGeKSJFXKEJckqVKGuCRJlTLEJUmq1P8HJbKzSPf47VIAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "dist = {}\n",
        "for _, row in train_df.iterrows():\n",
        "  length = len(row['answer'].split())\n",
        "  dist[length] = dist.get(length, 0) + 1\n",
        "\n",
        "sorted_keys = sorted(dist.keys())\n",
        "total = sum(dist.values())\n",
        "dist = {k: sum([dist[i] for i in sorted_keys if i < k])/total for k in sorted_keys}\n",
        "plt.plot(list(dist.keys()), list(dist.values()))\n",
        "plt.xticks([i for i in range(50, max(dist.keys()), 50)])\n",
        "plt.xlabel('answer length')\n",
        "plt.ylabel('Dataset percentage')\n",
        "plt.title('Percentage of the samples with answer length less than a certain value.');"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "tAZwCP6aEh8s"
      },
      "source": [
        "Basically more than 95% of the dataset has answers shorter than 50 tokens. Let's plot only these values then:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 513
        },
        "id": "PHMuVJGnEDmJ",
        "outputId": "f999d3c8-ac8d-431c-cb0b-69e610910742"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAfEAAAHwCAYAAAC2blbYAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3deXxcZ3n3/88lyfKeOIntJMTOvhF2yEpYAmUJS6GlhRKgD1sJ/bW0UKAtbSlbSynt07I8TSmB5oGyhZQ+lEBDKaUJCUtCEhICWRyMIXGcWHZix9qskSVdvz/OkT1RJHtkazRzpM/79ZI9Z5kz1xydma/uM/fcJzITSZJUPR2tLkCSJO0fQ1ySpIoyxCVJqihDXJKkijLEJUmqKENckqSKMsRFRPxqRGyMiP6IeEID658XEffMRm2zISKOjYiMiK4WPPZTI2LdXpa3rLZWKJ/riS143Gkd0xFxVUT8VjNr2sfjfyoi/rJVj9+OIuJPI+KTLXjclhyz4+ZMiEfELyJiZxlEPeVBvqzVdY2LiPdExGdbXccU/jfwpsxclpk3TVzY6oN0LsvMazLzlPHp8jh+Vitrmg+qdExHxGsi4jutrmO2TfcPpcz8q8xs2R9WrTJnQrz0y5m5DHgicDrwzuncOQpzbZ804hjg1lYXofljvpxZ0PTN4/fh/ZOZc+IH+AXwrLrpvwW+Vt4+G/ge8CDwI+C8uvWuAt4PfBfYCZwIPAr4JrAN6AH+tFy3A3gH8DPgAeAy4NBy2bFAAq8G7gbuB/6sXHY+MAzsAvqBH5XzXwvcDvQBG4A3TnhOfwTcB9wL/Fa5/RPLZQspWtB3lzX+E7B4in3TQfEHzV3AFuBfgIPLbfSX2x0AfjbJfa+uW94P/AZwHnAP8LZye/cBr627z3RqOxH4NrCj3GdfrFv2EWAj0AvcCDy1btl7gH8FPlvuvx8DJwN/Uta0EXjOhN/zB4AflNv7yiS/u65y+mDgn8vntQn4S6BzX/VOeF6fBt5W3j6q3P7vltMnUBxbHeP7spz/GWCM4jjsL3//47U97Lia4nFfANxUPseNwHvqlu11W8CZwA3lfXuAv5/OcymnXwjcTPFa+x7w2Amv0T8GbgFq4/t7Qv0NHePs+xg8DPhq+VyuL3+H39nfY3qSOq8Cfqtu+nUUr+XtwDeAY8r5AXyo3GYvxXH66HLZ84HbKI7fTcDbJ3mcRwJDwGhZ64Pl/E8BFwH/Ud7/OuCEabx2LqN4H+ij+AP+9L081ym3Ncm6i4G/o3iv2QF8p+53Np334c+Vz3mofN7/0ODz+mwjx/qEms8CNlO+xst5vwrcUve6+H5Z933APwDdUxyzE4+L11Aed+X0qezJlnXAy6bal43+tDx8Z+qHuhAH1pYH5l9QvOk8UL5gOoBnl9Or6nb63RTB3QUsL39RbwMWldNnleu+GbgWWEPxBvNx4AsTDppPlAfy4yjeqB458QCrq/kFFG+CATwdGASeWC47vzywHgUsoQir+oPlQ8DlwKFljV8FPjDFvnkdsB44HlgG/D/gM5MdhFPc/yHLKd7wRoD3AQvKfTsIHLIftX0B+LPyd7MIeErdsldRvBl3lb+PzcCiuv05BDy3XP4vwM/LbS0A3gD8fMKbxCbg0cBS4N94+At+PMS/XP5ulwKrKYL/jfuqd5J9/tXy9iso/vD7Yt2yr9Tty3smO44bOa4medzzgMeU9T2WIvx+pcFj9PvAb5a3lwFnT/O5PIEirM4COineQH8BLKx7bjdTvD6n+qOuoWOcfR+Dl5Y/S4DTKN74vzPZ4zSyvUnqvIryzRp4McXr65EUx+I7ge+Vy55LETYrKF7njwSOLJfdRxlCwCGUr/1JHus19bWX8z5F8T52ZvmYnwMuneZr5/nl7+kDwLV7ef1Pua1J1r2o3DdHldt+MsV75XTfhxcwIRAbfF4TX9ONvm5+Bjy7bvpfgXeUt59E8QdIV7nd24G3THHMPqTm+t8dxfvJRorGWxfF6+V+4LSp9n0jPy0P35n6oXiD6Kf4a+ku4B/LX94fUxdY5brfAF5dt9PfV7fsAuCmKR7jduCX6qaPpGhdj/9yE1hTt/wHwMsnHmB7eQ7/Dry5vH0JdcFH0QLM8v+gaEXU/+V9DnWhNWG73wJ+p276lPG6Jx6EU9x/sje8ndS1pCjevM/ej9r+Bbi4fr/tpY7twOPq9uc365b9cvn7H28xLy/rXlH3e/7ruvVPozg70ln3u+sCDqd4sS+uW/cC4Mrp1Evxx9l2ijesfwLeyJ4W96eBt9bty0ZCfNLjqoF99mHgQ41si6KF+l5g5X4+l48BfzHhvuuAp9c9t9fto96GjvF9HIOdFMf3KXXLdrfEp3tMT1HnVewJ8a8Dr69b1kHxB8AxwDOBO8u6OiZs4+5yXx60j33yGiYP8U/WTT8fuGMar53/nvBa2NnI8TRxWxPmd5T7cLJl03ofnriPp/G8JoZ4Q6+b8vi4pLy9vDz2jpli3bcAX57sWJpYMw8N8d8ArpmwrY8D725030/2M9c+d/iVzFyRmcdk5u9k5k6KF9JLI+LB8R/gKRQBPG5j3e21FH+VTeYY4Mt127md4pTP4XXrbK67PUjRoplURDwvIq6NiG3l9p4PrCwXP2JCXfW3V1G0MG6sq+U/y/mTeQTFHzbj7mJPYO2vBzJzpG56/LlOt7Y/onjD/kFE3BoRrxtfEBFvj4jbI2JHuZ2D2bN/oGhljtsJ3J+Zo3XT8ND9X78P76L4a79+e1D8jhcA99XV/3GKFvle662XmT+jeCN4PPBU4GvAvRFxCsVZl29Pvjum1NBxFRFnRcSVEbE1InYAvz3Jc5xqW6+n+Ejijoi4PiJeOM3ncgzwtgmvtbUUx9+4+t/B3jRyHO3tGOxi6tfPVKba3r4cA3ykrs5tFMfIUZn5PxSnXy8CtkTExRFxUHm/X6N4zd8VEd+OiHMaeKx6Ux4TDbx2Jt530VT9FBrY1riVFGenJnv/nO778KSmUcu4Rt+PPw+8JCIWAi8BfpiZd5WPeXJEfC0iNkdEL/BX+3jMqRwDnDVhH7wSOGI/trXbfOhcspHiL8A37GWdnLD+y/eyrddl5ncnLoiIY/dRR/1jUB4s/wb8L4rTkbsi4t8pXvxQnGpbU3eXtXW376cIqUdl5qZ9PC4Un6kfUzd9NMWpw57JVz8g06otMzdTnPomIp4C/HdEXE3x4v4j4JeAWzNzLCK2s2f/7I/6fXg0RWvt/gnzN1K0xFdOeEPfa72ZuX6Sx/s28OsUn59tiohvU5xiPoTitPJkcor5jfo8RWg8LzOHIuLDNPiGk5k/BS4oOxW9BPhSRByWmQMNPpeNwPsz8/17e5gGn8d0j/F6WymO7zUUrWB46O94po0/789NtjAzPwp8NCJWU3wW/YfAn2fm9cCLI2IB8KZy2WR1TuuYiIinMkOvnWlu636K0/QnUHzmXW+678MPm57J5/WwB868LSLuAp5H8ZHR5+sWf4yin8kFmdkXEW+heC1MZoDij89x9QG9Efh2Zj77QOutN9da4pP5LPDLEfHciOiMiEVRfCd0zRTrfw04MiLeEhELI2J5RJxVLvsn4P0RcQxARKyKiBc3WEcPcGxdr8tuis+KtgIjEfE84Dl1618GvDYiHhkRS4A/H1+QmWMUn/V8qHxjICKOiojnTvHYXwD+ICKOi+Jrd39F8Znmw0JqL7Uf38iK060tIl5a97vYTvHCHaM4pTVCsX+6IuJdwEGTbWMaXhURp5X7833Al+pa7uP13wf8F/B3EXFQRHRExAkR8fR91DuZb1O8OV9dTl9VTn9n4uPWaXhfT2E5sK0M8DMp3pAaEhGviohV5e/wwXL2+HNr5Ll8Avjt8mxARMTSiHhBRCyf7pPYj2O8/r6jFP0+3hMRSyLiVIo/lusd6H6u90/An0TEo8o6D46Il5a3zyj3xwKKN/ghYCwiuiPilRFxcGbuouioNdVx1AOsiYjuBuuZyddOw9sqf2eXAH8fEY8o32/PKRss030fhof/jprxnlDv8xT9np5G8Zl4/eP2Av3lsfT/7WUbN1O06JdE8RXG19ct+xpwckT8ZkQsKH/OiIhHHkjRcz7EM3MjRceTP6X45W+k+Et40ueemX0UnS5+meJUzE+BZ5SLP0LR0ea/IqKPopPbWZNtZxLjB8UDEfHD8nF+nyKst1O82V5eV8fXgY8CV1J0mrm2XFQr///j8fnlKZ7/pvisezKXUPR8vpqi89cQ8HsN1g3FZ02fLk8BvayB9adT2xnAdRHRT/H835yZGyg+L/tPipbUXWXNjZ6KncpnKD5L3Exx2u/3p1jvf1H8kXUbxe/mS+w57TdVvZP5NsUbwHjwfYfir/Srp1gfik5G7yz39dsbeE4T/Q7wvvL4fBfF8dWo84Fby+f2EYrPD8c/ltjnc8nMGyjOUvwDxX5bT/GZ4P6aznE00ZsoTrVupvi9f4E9rx2Y/jE9pcz8MvBB4NKyzp9QtOigCJlPUOyPuyg6c/1tuew3gV+U9/ltilOrk/kfio66myPi/gZKmsnXznS39XaKHvjXU3ys8EGKvgDTeh8ufQT49YjYHhEfneHnNZkvUHw89D+ZWb+f307x/txH8bv84l628SGKvjY9FP1Fdp+dKd/zn0NxpvdeimPzgxSNufHBar4+3aKj/HBdba78a+0nFD19G21BqxQRV1F0epn1EZ3UehHxQeCIzHx1q2uRZtKcb4lXWRTDoS6MiEMo/mL7qgEu7VtEnBoRjy1P659JcVrzy62uS5pphnh7eyPF11x+RtELfm+fxUjaYznF5+IDFKc//45igB9pTvF0uiRJFWVLXJKkijLEJUmqqMoN9rJy5co89thjW12GJEmz4sYbb7w/Mycd9bJyIX7sscdyww03tLoMSZJmRTma3KQ8nS5JUkUZ4pIkVZQhLklSRRnikiRVlCEuSVJFGeKSJFWUIS5JUkUZ4pIkVZQhLklSRRnikiRVlCEuSVJFGeKSJFWUIS5JUkUZ4pIkVVTTQjwiLomILRHxkymWR0R8NCLWR8QtEfHEZtUiSdJc1MyW+KeA8/ey/HnASeXPhcDHmliLJElzTlezNpyZV0fEsXtZ5cXAv2RmAtdGxIqIODIz72tWTZLUTMXb2fjtCcv2su6U23vI+uPzpn6MVtmfOvJhe2T6jzHdfToTu6uR5xoBBy1aMAOPtm9NC/EGHAVsrJu+p5xniGtOGR1Ldo2OMTw6xq6RMXaN7pkeHhljZDTZNTa2e73RsWRkNBkZS0bHivXHspg3OpaMZrFsbGzC/+PrZHG/ieuMZjKWMFauOzpWvOmNZTKaMJYJ5f9jmWTCWBbrJOPz99xnbKx4I94zr9wGxRtdljeSPW98YxNqGKu73+jY+GNmef89NUDu3mZO2Ob4dDGxZ51y8iHht+f2PuSekNn9XCZ5bGkyBy3q4pb3PHdWHquVId6wiLiQ4pQ7Rx99dIur0VySmQztGqO/NlL8DI3QV9tF/1AxvXPXKLVdYwyNjDK0a4zarlFqI2MM7Rotf8bYuWuUneX0zuHx28U6O3eNMjo2++/4XR1BR0fQ1RF0RnG7syPoiKAj2H07JtweXx7smZ74f0fddIxPd3TsXj4uIgiKVknUTXd07NlGR/nYnbu3WT5+WUNHB1DWsmdb8ZBt7nm88v+65ePzx9eL3f8U6+3Nw7ZRTu2eX//YE+63Z/5DHyMmPOTeK5hkexM3MMk298e+9kVD29iPTUz3LpM9xr72cTNM9nuo1901e33GWxnim4C1ddNrynkPk5kXAxcDnH766f4NrEllJn21Ebb21djaV2P7wDDbBod5cHAX2waG2T44XM7bxYPl/P7ayLRCtrurg0VdHSxa0MmiBZ0s7OpgSXdx+9Cl3Sxe0cniBZ0s6i7/X9BBd2cnC7qC7s4Ours6WNA5/lPM6+rsoKuzCNyujslvd9b9dHV00NEBXR0dZUCXt3cH9b7fZCTNDa0M8cuBN0XEpcBZwA4/D9dUhnaNcu+DO9n04E42bd/JvTuG2NpX4/7+2u7Q3tpfY3hkbNL7L+3uZMWSbg5d2s2KJQs49rAlHLx4AcsXdbFs4QKWLepi+cIuli7sYtnCLpYvKm6PB/GiBZ10d3bQ0WE4SmofTQvxiPgCcB6wMiLuAd4NLADIzH8CrgCeD6wHBoHXNqsWtb/M5IGBYdZv6ednW/u5e9sg92wvAnvTgzvZ2ld7yPoRcNjSblYuW8iq5Qs5ftVSVpW3Vy1fyMplCzl06Z7QXtjV2aJnJknN08ze6RfsY3kCv9usx1d7ykzu3THET3v6dgf2+i3Fz/bBXbvX6+7s4BErFrHmkCU885TVHHXIYtYcspijVizmqEMWc8RBi+jqdKwiSfNbJTq2qboyk/Vb+rn259v4wc+3cd2GB9hS16o+ZMkCTlq9nPMffSQnrl62++fIgxZ56lqS9sEQ14waG0tu39xbBvY2fvCLbWwbGAbg8IMWcvbxh3H6sYdwyuHLOXH1Mg5btrDFFUtSdRniOmCZyU0bH+Tym+/la7fcy/39RWivPXQxzzhlNWcdfyhnHXcoRx+6xF7TkjSDDHHtt/Vb+vjKzffylZvv5e5tg3R3dfCsR67m2acdzlnHHcYjVixudYmSNKcZ4pqW+3bs5Ks/upd/v+lebruvl46Ac09cye8980Se++gjZm2oQUmSIa4GZCbX/2I7n7xmA9+8vYdMeNzaFbzrhafxwscdyerli1pdoiTNS4a4pjQyOsYVP9nMP1+zgR/ds4MVSxbwO+edwEuftJZjVy5tdXmSNO8Z4nqY3qFdfPEHG/nU937Bpgd3cvzKpfzlrzyaX3viGhZ3O2iKJLULQ1y73bdjJ5+85ud88fqN9NdGOOu4Q3nvix7FM09d7Xe2JakNGeJibCz53A/u5gNX3E5tZIwXPvZIfuspx/OYNQe3ujRJ0l4Y4vPcxm2D/NGXbuH7Gx7gqSet5K9+9TGsPXRJq8uSJDXAEJ+n6lvfHRF84CWP4eVnrHUwFkmqEEN8HprY+v7rX3ssRzkwiyRVjiE+j4yNJZ+77i4+8PU76Ijgr1/yGH7D1rckVZYhPk9sGxjmdz/3Q76/4QGedvIqPvCSx9j6lqSKM8TngQf6a7zyk9fx8/sHbH1L0hxiiM9x9QH+z68+g6ectLLVJUmSZoghPofd31/jlZ+4jru2DXDJa87g3BMNcEmaSwzxOer+/hqv+MS13L1tkEtefQZPNsAlac4xxOegrX1FgG/cPsglrzmDJ59ggEvSXGSIzzFb+oZ4xSeuY9P2nfzf15zJOScc1uqSJElNYojPIVv6hrjg4mu598Eh/u9rz+Ds4w1wSZrLDPE5YkvvEBd8wgCXpPnEEJ8Ddo2O8YbP3Mh9O4b41GvP4CwDXJLmBUN8Dvjot37KjzY+yEWveKIBLknzSEerC9CBuf4X27joyvX82hPX8ILHHtnqciRJs8gQr7DeoV38wRdvZs0hS3jPi05rdTmSpFnm6fQKe89XbuW+HUNc9sZzWL5oQavLkSTNMlviFfXVH93L/7tpE296xok86ZhDWl2OJKkFDPEKuvfBnfzZl3/ME45ewe8988RWlyNJahFDvGJGx5K3XnYzo2PJh3/j8XR1+iuUpPnKBKiYT1yzgWs3bOPdL3oUxxy2tNXlSJJayBCvkJ9s2sHf/dc6nvfoI3jpk9a0uhxJUosZ4hWxc3iU37/0Jg5d2s1f/epjiIhWlyRJajG/YlYR77/iNjZsHeCzrz+LQ5Z2t7ocSVIbsCVeAT/ZtIPPXns3r3/KcTzlJK8NLkkqGOIVcPHVG1i2sIs3P+ukVpciSWojhnibu2f7IP/x4/u44My1HOSobJKkOoZ4m7vkO78ggNeee1yrS5EktRlDvI3tGNzFpdffzS8/7hE8YsXiVpcjSWozhngb++x1dzE4PMobnnp8q0uRJLUhQ7xN1UZG+dT3fsFTT1rJaY84qNXlSJLakCHepr5y071s7atx4dNshUuSJmeIt6GxseTiazbwyCMP4ikn+r1wSdLkDPE2dNWdW1i/pZ8Ln3acw6tKkqZkiLehj397A0cevIgXPvYRrS5FktTGDPE286OND3Ldz7fxunOPY4HXCpck7YUp0WYuvmYDyxd28fIz17a6FElSmzPE28jdDwzy9R/fxyvOPprlDrEqSdoHQ7yNXPLdn9PZEbz2yQ6xKknaN0O8TWwfGOaL12/kRY87iiMOXtTqciRJFWCIt4nPXXcXO3eN8oan2QqXJDXGEG8DQ7tG+dT37uLpJ6/i1CMcYlWS1BhDvA185eZN3N/vEKuSpOkxxNvA1265j+NXLeXJJxzW6lIkSRViiLfYQG2E6zZs41mPPNwhViVJ02KIt9h319/P8OgY552yqtWlSJIqxhBvsSvXbWXZwi5OP+bQVpciSaoYQ7yFMpOr1m3hqSetpLvLX4UkaXpMjha6Y3Mf9+0Y4hmnrG51KZKkCjLEW+jKdVsA/DxckrRfDPEWuvKOLTz6qINYfZDDrEqSps8Qb5Edg7u48a7tnkqXJO03Q7xFrv7pVsYSzjPEJUn7yRBvkSvv2MIhSxbw+LUrWl2KJKmiDPEWGBtLrrpzK08/eRWdHY7SJknaP4Z4C9yyaQfbBoZ5xqmeSpck7T9DvAX+544tdAQ87SS/WiZJ2n+GeAtctW4LTzj6EA5Z2t3qUiRJFWaIz7KtfTVuuWcHz3CAF0nSAWpqiEfE+RGxLiLWR8Q7Jll+dERcGRE3RcQtEfH8ZtbTDq7aPUqbn4dLkg5M00I8IjqBi4DnAacBF0TEaRNWeydwWWY+AXg58I/NqqddXLVuK6uXL+RRjzio1aVIkiqumS3xM4H1mbkhM4eBS4EXT1gngfE0Oxi4t4n1tNyu0TGu/ulWnnHKaiL8apkk6cB0NXHbRwEb66bvAc6asM57gP+KiN8DlgLPamI9LXfjXdvpGxrhGaf6ebgk6cC1umPbBcCnMnMN8HzgMxHxsJoi4sKIuCEibti6deusFzlTrly3hQWdwbknrmx1KZKkOaCZIb4JWFs3vaacV+/1wGUAmfl9YBHwsITLzIsz8/TMPH3Vquq2Yq+6YytnHHsoyxctaHUpkqQ5oJkhfj1wUkQcFxHdFB3XLp+wzt3ALwFExCMpQry6Te292PTgTtb19HnVMknSjGlaiGfmCPAm4BvA7RS90G+NiPdFxIvK1d4GvCEifgR8AXhNZmazamqlK+8ovlrmUKuSpJnSzI5tZOYVwBUT5r2r7vZtwLnNrKFdXLVuC2sPXcwJq5a2uhRJ0hzR6o5t88LQrlG+u/4Bv1omSZpRhvgsuO7n29i5a9RT6ZKkGWWIz4Ir79jCwq4Ozjn+sFaXIkmaQwzxWXDlui08+YTDWLSgs9WlSJLmEEO8ybb21bjrgUEHeJEkzThDvMnu7OkD4LQjveCJJGlmGeJNdsfmIsRPPmJ5iyuRJM01hniT3bm5j5XLulm5bGGrS5EkzTGGeJPd0dPHyYfbCpckzTxDvInGxpKf9vRxiqfSJUlNYIg30T3bdzI4PMoptsQlSU1giDfRHZt7AWyJS5KawhBvovGvl51kS1yS1ASGeBPdsbmPtYcuZtnCpl4sTpI0TxniTXRnT5+fh0uSmsYQb5LhkTE2bB3w83BJUtMY4k3ys639jIyl3xGXJDWNId4k453aTj3CMdMlSc1hiDfJHZv7WNAZHLdyaatLkSTNUYZ4k9y5uY/jVy6ju8tdLElqDhOmSe7Y7HCrkqTmMsSboG9oF5se3GmIS5KayhBvgjt7+gH8jrgkqakM8SYY75luS1yS1EyGeBOs29zH0u5OjlqxuNWlSJLmMEO8Ce7Y3MtJhy+noyNaXYokaQ4zxGdYZrJucx+neipdktRkhvgM29pfY/vgLodblSQ1nSE+w+7cXPRMtyUuSWo2Q3yG3bG5F4CTDXFJUpMZ4jPszp4+Vi7rZuWyha0uRZI0xxniM2ydw61KkmaJIT6DxsaSO3v67dQmSZoVhvgM2rh9kJ27Ru3UJkmaFYb4DLpjczHcqi1xSdJsMMRn0J2GuCRpFhniM+iOnj7WHrqYpQu7Wl2KJGkeMMRn0J2b+zjl8INaXYYkaZ4wxGdIbWSUDfcPcMoRy1pdiiRpnjDEZ8iGrQOMjiWnHGFLXJI0OwzxGbKu7NR2ip3aJEmzxBCfIet6+ljQGRy3cmmrS5EkzROG+AxZt7mP41cuo7vLXSpJmh0mzgxxzHRJ0mwzxGdA39AuNj240xCXJM0qQ3wG3NljpzZJ0uwzxGfAus39ALbEJUmzyhCfAes297K0u5OjVixudSmSpHnEEJ8B63r6OPmI5XR0RKtLkSTNI4b4AcrMome6n4dLkmaZIX6AtvbV2D64y8/DJUmzzhA/QOvsmS5JahFD/ADd2VP0TD/ZlrgkaZYZ4gdo0/adLOnu5LCl3a0uRZI0zxjiB6inb4jDD1pEhD3TJUmzyxA/QFt6h1i9fGGry5AkzUOG+AHq6a1x+EGLWl2GJGkeMsQPQGbS0zvEEQcb4pKk2WeIH4DenSPURsY8nS5JaglD/AD09A0BeDpdktQShvgB6Ok1xCVJrWOIH4DNO8ZD3NPpkqTZZ4gfgC19NQBWL7clLkmafYb4AejpHeKgRV0s7u5sdSmSpHnIED8APb1Dfh4uSWqZhkI8Ip4SEa8tb6+KiOOaW1Y19PTW/I64JKll9hniEfFu4I+BPylnLQA+28yiqqIYctUQlyS1RiMt8V8FXgQMAGTmvcC8v+7m2Fiypa9mz3RJUss0EuLDmZlAAkTE0uaWVA3bBocZGUs/E5cktUwjIX5ZRHwcWBERbwD+G/hEc8tqf35HXJLUavsM8cz838CXgH8DTgHelZn/p5GNR8T5EbEuItZHxDumWOdlEXFbRNwaEZ+fTvGttKUccnW1LXFJUot0NbJSZn4T+OZ0NhwRncBFwLOBe4DrI+LyzLytbp2TKDrMnZuZ2yNi9XQeo5V6eouBXjydLklqlUZ6p/dFRO+En40R8eWIOH4vdz0TWJ+ZGzJzGLgUePGEdd4AXJSZ2wEyc8v+PpHZNj5u+qplnk6XJLVGIy3xD4I1GusAABtoSURBVFO0pD8PBPBy4ATgh8AlwHlT3O8oYGPd9D3AWRPWORkgIr4LdALvycz/bLD2lurprbFyWTfdXY6XI0lqjUZC/EWZ+bi66Ysj4ubM/OOI+NMZePyTKP4QWANcHRGPycwH61eKiAuBCwGOPvroA3zImeF3xCVJrdZIM3Kw7HzWUf68DBgql+Ve7rcJWFs3vaacV+8e4PLM3JWZPwfupAj1h8jMizPz9Mw8fdWqVQ2U3Hw9fUP2TJcktVQjIf5K4DeBLUBPeftVEbEYeNNe7nc9cFJEHBcR3RSn4S+fsM6/U56Oj4iVFKfXN0znCbRKT2/NTm2SpJba5+n0zNwA/PIUi7+zl/uNRMSbgG9QfN59SWbeGhHvA27IzMvLZc+JiNuAUeAPM/OB6T6J2bZrdIz7+2t+vUyS1FL7DPGIWAS8HngUsDu1MvN1+7pvZl4BXDFh3rvqbifw1vKnMu7vr5HpQC+SpNZq5HT6Z4AjgOcC36b4bLuvmUW1u93fEbdjmySphRoJ8RMz88+Bgcz8NPACHv5VsXll/DviXoZUktRKjYT4rvL/ByPi0cDBQGVGVmuGLb3jQ656Ol2S1DqNfE/84og4BHgnRe/yZcCfN7WqNtfTW6OzIzhsqSEuSWqdRkL8W+WwqFcDxwNExHFNrarN9fQOsWrZQjo7otWlSJLmsUZOp//bJPO+NNOFVElPX82e6ZKklpuyJR4Rp1J8rezgiHhJ3aKDqPuq2XzUs2OIow9b0uoyJEnz3N5Op58CvBBYwUMHe+mjuPrYvNXTN8QZxx3S6jIkSfPclCGemV8BvhIR52Tm92exprY2tGuUBwd3+R1xSVLLNdKxbX15tbJj69dvZMS2uWhrXznQi98RlyS1WCMh/hXgGuC/KcY3n9fGB3rx4ieSpFZrJMSXZOYfN72Sitg95Kq90yVJLdbIV8y+FhHPb3olFbG7Je5n4pKkFmskxN9MEeRDEdEbEX0R0dvswtpVT98Q3Z0drFiyoNWlSJLmuUauJ758Ngqpip4dQ6w+aCERjtYmSWqtfbbEo/CqiPjzcnptRJzZ/NLaU09vzU5tkqS20Mjp9H8EzgFeUU73Axc1raI219M3xBGGuCSpDTQS4mdl5u8CQwDlxVC6m1pVG9vSW/MSpJKkttDQ9cQjohNIgIhYBYw1tao21V8bob824ul0SVJbaCTEPwp8GVgdEe8HvgP8VVOralNbdg/0YktcktR6jfRO/1xE3Aj8EhDAr2Tm7U2vrA3tHujF74hLktrAPkM8Is4Gbs3Mi8rpgyLirMy8runVtZktfUVLfLWn0yVJbaCR0+kfo+iRPq6/nDfvbN7h6XRJUvtoJMQjM3N8IjPHaGzM9Tmnp7fGku5Oli2cl09fktRmGgnxDRHx+xGxoPx5M7Ch2YW1o/HviDtamySpHTQS4r8NPBnYBNwDnAVc2Myi2tWW3iG/Iy5Jaht7PS9cfj/8Q5n58lmqp6319NZ4wtErWl2GJEnAPlrimTkKHBMR83aEtnGZSU/vkAO9SJLaRiM9tDYA342Iy4GB8ZmZ+fdNq6oN9e4coTYyxurlnk6XJLWHRkL8Z+VPBzBvL0u6efdobbbEJUntoZER294LEBFLMnOw+SW1px5DXJLUZhq5nvg5EXEbcEc5/biI+MemV9ZmxkPcy5BKktpFI18x+zDwXOABgMz8EfC0ZhbVjrb0FeOm+xUzSVK7aCTEycyNE2aNNqGWttbTO8TBixewaEFnq0uRJAlorGPbxoh4MpARsQB4MzDvrmJWfL3MVrgkqX00OmLb7wJHAfcCjy+n55We3pqd2iRJbaWR3un3A6+chVra2pbeIU5YtbLVZUiStFsjvdOPj4ivRsTWiNgSEV+JiONno7h2MTaWbOmreTpdktRWGjmd/nngMuBI4BHAvwJfaGZR7eaBgWFGxtLT6ZKkttJIiC/JzM9k5kj581lgXqWZA71IktpRI73Tvx4R7wAuBRL4DeCKiDgUIDO3NbG+trClbzzEPZ0uSWofjYT4y8r/3zhh/sspQn3Ofz7e01sM9GJLXJLUThrpnX7cbBTSzsZPp6/yCmaSpDbS0Iht811Pb42Vy7pZ0OnukiS1D1OpAVt6h1i93FPpkqT2Yog3YLNDrkqS2lAjg718q5F5c1lPb40jDrYlLklqL1N2bIuIRcASYGVEHAJEuegginHU54Vdo2M8MFDzdLokqe3srXf6G4G3UIzS9sO6+b3APzSzqHZyf3+NTL9eJklqP1OGeGZ+BPhIRPxeZv6fWayprez5jrifiUuS2ksjHdsuiYh3RsTFABFxUkS8sMl1tQ2HXJUktauGQhwYBp5cTm8C/rJpFbWZLWWIr7YlLklqM42E+AmZ+TfALoDMHGRPJ7c5r6e3RmdHcNhSQ1yS1F4aCfHhiFhMMU46EXECUGtqVW1kc+8Qq5YtpLNj3vzdIkmqiEYugPJu4D+BtRHxOeBc4DXNLKqd9PQOcbjfEZcktaFGLoDyzYj4IXA2xWn0N2fm/U2vrE1s6a1xzGFLWl2GJEkP08iIbecCQ5n5H8AK4E8j4pimV9YmtvQNefUySVJbauQz8Y8BgxHxOOCtwM+Af2lqVW2kvzbCQYsXtLoMSZIeppEQH8nMBF4MXJSZFwHLm1tWexgeGWPXaLK0u7PVpUiS9DCNdGzri4g/AV4FPC0iOoB50TQdHB4BYOnCRnaTJEmzq5GW+G9QfKXs9Zm5GVgD/G1Tq2oT/bUyxLsNcUlS+2mkd/pm4O/rpu9mnnwmPjg8CsCShZ5OlyS1n0Z6p58dEddHRH9EDEfEaETsmI3iWm2g5ul0SVL7auR0+j8AFwA/BRYDvwX8YzOLahcDtaIl7ul0SVI7aiTEycz1QGdmjmbm/wXOb25Z7WGg7Ni2xN7pkqQ21EgTczAiuoGbI+JvgPtoMPyrbrx3+jJPp0uS2lAjYfyb5XpvAgaAtcBLmllUu+iv2bFNktS+GgnxX8nMoczszcz3ZuZbgRc2u7B2MOhXzCRJbayREH/1JPNeM8N1tKWB4VEiYPECW+KSpPYzZRMzIi4AXgEcFxGX1y1aDmxrdmHtYKA2wpIFnXR4LXFJUhva23ni71F0YlsJ/F3d/D7glkY2HhHnAx8BOoFPZuZfT7HerwFfAs7IzBsa2fZsGBweYYmd2iRJbWrKhMrMu4C7gHP2Z8MR0QlcBDwbuAe4PiIuz8zbJqy3HHgzcN3+PE4zDdRG7ZkuSWpb+ztiW28D2z4TWJ+ZGzJzGLiU4kpoE/0F8EFgaFqVz4KB2ojfEZckta39HbHtogbudxSwsW76nnLebhHxRGBtZv5HQ9XOsoHhEXumS5LaVstGbCsvafr3wNsaWPfCiLghIm7YunXrgT50wwaHR1nqd8QlSW2qkRB/yIhtEfEHDd5vE8XAMOPWlPPGLQceDVwVEb8AzgYuj4jTJ24oMy/OzNMz8/RVq1Y18NAzo79mxzZJUvva3xHbfq2B+10PnBQRx5V/BLwc2P1VtczckZkrM/PYzDwWuBZ4UVv1Tq+NstTPxCVJbaqR64nfFRGrytvvbXTDmTkSEW8CvkHxFbNLMvPWiHgfcENmXr73LbTewPCIlyGVJLWtvQ32EsC7KVrgHeWsEeD/ZOb7Gtl4Zl4BXDFh3rumWPe8BmueFZnJQM2ObZKk9rW30+l/AJxLMQDLoZl5CHAWcG75uficVhsZYyy9+IkkqX3tLcR/E7ggM38+PiMzNwCvAv5XswtrtYGalyGVJLW3vYX4gsy8f+LMzNwKLGheSe1hYPwypJ5OlyS1qb2F+PB+LpsTBobHL0Pq6XRJUnvaWzPzcVMMrxrAoibV0zYGx0Pc0+mSpDa1twugzOsmaH95Ot0R2yRJ7aqhYVfno8GyY5ufiUuS2pUhPoWB4aIlbu90SVK7MsSnMLC7Je7pdElSezLEpzBgxzZJUpszxKcwWBulsyNY2OUukiS1JxNqCv21EZZ0d1IMIS9JUvsxxKcwODxipzZJUlszxKcwMDxqpzZJUlszxKcwUPNa4pKk9maIT2GwNuq1xCVJbc0Qn0J/bcQhVyVJbc0Qn8Lg8IhDrkqS2pohPoWB4VE/E5cktTVDfAoDtRGvJS5JamuG+CTGxpLB4VGW2BKXJLUxQ3wSO3eNX8HMlrgkqX0Z4pMY8FrikqQKMMQnMX4tcb9iJklqZ4b4JMZb4g72IklqZ4b4JHaHuB3bJEltzBCfxGB5Ot0LoEiS2pkhPomB4aIl7qVIJUntzBCfxO7e6Ya4JKmNGeKTGKiVvdM9nS5JamOG+CQGh/2euCSp/Rnik+ivjdLd2UF3l7tHktS+TKlJDA6PsMSBXiRJbc4Qn8RAbdSBXiRJbc8Qn8RAbcQhVyVJbc8Qn8TA8Iid2iRJbc8Qn8Tg8KgDvUiS2p4hPomB2ohDrkqS2p4hPomB4REvfiJJanuG+CQGa6N2bJMktT1DfBL9tRG/YiZJanuG+AQjo2PURsbsnS5JanuG+ASDu8qLn3g6XZLU5gzxCcYvQ2rHNklSuzPEJxi/DKlfMZMktTtDfILxy5A62Iskqd0Z4hP017yWuCSpGgzxCQZrdmyTJFWDIT7BwLAd2yRJ1WCITzDesc3BXiRJ7c4Qn2C8Y9sST6dLktqcIT6BLXFJUlUY4hMMDI+waEEHnR3R6lIkSdorQ3yCAS9+IkmqCEN8gsHhUXumS5IqwRCfoL824pCrkqRKMMQnGBwesSUuSaoEQ3yCgZqn0yVJ1WCIT1B0bPN0uiSp/RniEwwOj3rxE0lSJRjiEwwMj7DM0dokSRVgiE8wUBthiZ+JS5IqwBCvMzwyxq7RZJkhLkmqAEO8zu6Ln9ixTZJUAYZ4nf5aeS1xO7ZJkirAEK8zOFxewczT6ZKkCjDE6wzUvJa4JKk6DPE6XktcklQlhnidgbJj21Jb4pKkCjDE64z3TrclLkmqgqaGeEScHxHrImJ9RLxjkuVvjYjbIuKWiPhWRBzTzHr2pb88ne5n4pKkKmhaiEdEJ3AR8DzgNOCCiDhtwmo3Aadn5mOBLwF/06x6GjFYdmxzsBdJUhU0syV+JrA+Mzdk5jBwKfDi+hUy88rMHCwnrwXWNLGefRoYHiUCFnXZEpcktb9mhvhRwMa66XvKeVN5PfD1JtazTwO1EZYs6KSjI1pZhiRJDWmL88YR8SrgdODpUyy/ELgQ4Oijj25aHYPDIw70IkmqjGa2xDcBa+um15TzHiIingX8GfCizKxNtqHMvDgzT8/M01etWtWUYqH4nrghLkmqimaG+PXASRFxXER0Ay8HLq9fISKeAHycIsC3NLGWhgzURrz4iSSpMpoW4pk5ArwJ+AZwO3BZZt4aEe+LiBeVq/0tsAz414i4OSIun2Jzs2LA0+mSpAppamJl5hXAFRPmvavu9rOa+fjTNTg8ymFLu1tdhiRJDXHEtjr9tRGW2BKXJFWEIV5nsDbKModclSRVhCFeZ2B4xCFXJUmVYYiXMpOB2ogXP5EkVYYhXqqNjDGW2DtdklQZhnhpoOa1xCVJ1WKIlwbGL0Pq6XRJUkUY4qWB4fHLkNoSlyRVgyFeGixD3Ja4JKkqDPFSf3k63c/EJUlVYYiXBnd3bLMlLkmqBkO8NDBctsQ9nS5JqghDvDT+FTMvRSpJqgpDvDTeO93T6ZKkqjDES4O1UTo7goVd7hJJUjWYWKX+2ghLujuJiFaXIklSQwzx0uDwCMs8lS5JqhBDvDQwPGqnNklSpRjipYHaiJ3aJEmVYoiXBmujfkdcklQphnhpYHjEIVclSZViiJcGaiNe/ESSVCmGeGlgeNTPxCVJlWKIlwZrIyy1d7okqUIMcWBsLIuvmNkSlyRViCEO7NxVXMFsmR3bJEkVYoiz5+IndmyTJFWJIQ4M1MpridsSlyRViCHOnmuJO9iLJKlKDHFgcHi8JW6IS5KqwxBnT0vcC6BIkqrEEGdPxzYvRSpJqhJDnOLiJ4DfE5ckVYohDvTv7tjm6XRJUnUY4sCg3xOXJFWQIQ7010bp7uygu8vdIUmqDlOLoiXuQC+SpKoxxClGbPNUuiSpagxxiu+J2xKXJFWNIU7xPXFHa5MkVY0hTjHsquOmS5KqxhCnOJ3ukKuSpKoxxClOpzvkqiSpagxximFXl9ixTZJUMYY4xbCrfiYuSaqaeR/iI6Nj1EbG7J0uSaqceR/ig7vKK5jZsU2SVDHzPsQHxq9gZktcklQxhnh5LXFDXJJUNfM+xMcvQ+q1xCVJVTPvQ7y/5rXEJUnVNO9DfLA8ne5gL5Kkqpn3IT5Qnk53sBdJUtUY4uMd2zydLkmqmHkf4rs7ttkSlyRVzLwP8fGWuB3bJElVY4gPj7BoQQedHdHqUiRJmhZDvOZlSCVJ1TTvQ3xweNRT6ZKkSpr3Id5fG/HiJ5KkSpr3IT447Ol0SVI1zfsQH6iNssQQlyRVkCFeG/HiJ5KkSpr3IT44POplSCVJlTTvQ3xg2Ja4JKmaDPHaiJ+JS5IqaV6H+PDIGLtG097pkqRKmtchPn7xE78nLkmqonkd4v218gpmjtgmSaqgeR3ig8PltcQ9nS5JqqCmhnhEnB8R6yJifUS8Y5LlCyPii+Xy6yLi2GbWM9FA2RJf4rXEJUkV1LQQj4hO4CLgecBpwAURcdqE1V4PbM/ME4EPAR9sVj2TGb+WuKfTJUlV1MyW+JnA+szckJnDwKXAiyes82Lg0+XtLwG/FBGzdmHvgbJj21Jb4pKkCmpmiB8FbKybvqecN+k6mTkC7AAOa2JNDzHeO92WuCSpiirRsS0iLoyIGyLihq1bt87YdpctXMDj1hzM8kWGuCSpepqZXpuAtXXTa8p5k61zT0R0AQcDD0zcUGZeDFwMcPrpp+dMFfjs0w7n2acdPlObkyRpVjWzJX49cFJEHBcR3cDLgcsnrHM58Ory9q8D/5OZMxbSkiTNZU1riWfmSES8CfgG0Alckpm3RsT7gBsy83Lgn4HPRMR6YBtF0EuSpAY09cPgzLwCuGLCvHfV3R4CXtrMGiRJmqsq0bFNkiQ9nCEuSVJFGeKSJFWUIS5JUkUZ4pIkVZQhLklSRRnikiRVlCEuSVJFGeKSJFWUIS5JUkUZ4pIkVZQhLklSRRnikiRVlCEuSVJFGeKSJFVUZGara5iWiNgK3DWNu6wE7m9SOfOR+3PmuU9nnvt0Zrk/W+uYzFw12YLKhfh0RcQNmXl6q+uYK9yfM899OvPcpzPL/dm+PJ0uSVJFGeKSJFXUfAjxi1tdwBzj/px57tOZ5z6dWe7PNjXnPxOXJGmumg8tcUmS5qQ5G+IRcX5ErIuI9RHxjlbXU0URcUlEbImIn9TNOzQivhkRPy3/P6SVNVZJRKyNiCsj4raIuDUi3lzOd5/up4hYFBE/iIgflfv0veX84yLiuvL1/8WI6G51rVUSEZ0RcVNEfK2cdn+2qTkZ4hHRCVwEPA84DbggIk5rbVWV9Cng/Anz3gF8KzNPAr5VTqsxI8DbMvM04Gzgd8vj0n26/2rAMzPzccDjgfMj4mzgg8CHMvNEYDvw+hbWWEVvBm6vm3Z/tqk5GeLAmcD6zNyQmcPApcCLW1xT5WTm1cC2CbNfDHy6vP1p4FdmtagKy8z7MvOH5e0+ijfJo3Cf7rcs9JeTC8qfBJ4JfKmc7z6dhohYA7wA+GQ5Hbg/29ZcDfGjgI110/eU83TgDs/M+8rbm4HDW1lMVUXEscATgOtwnx6Q8tTvzcAW4JvAz4AHM3OkXMXX//R8GPgjYKycPgz3Z9uaqyGuWZDFVxv8esM0RcQy4N+At2Rmb/0y9+n0ZeZoZj4eWENxFu7UFpdUWRHxQmBLZt7Y6lrUmK5WF9Akm4C1ddNrynk6cD0RcWRm3hcRR1K0ftSgiFhAEeCfy8z/V852n86AzHwwIq4EzgFWRERX2Xr09d+4c4EXRcTzgUXAQcBHcH+2rbnaEr8eOKnsUdkNvBy4vMU1zRWXA68ub78a+EoLa6mU8rPFfwZuz8y/r1vkPt1PEbEqIlaUtxcDz6boa3Al8Ovlau7TBmXmn2Tmmsw8luJ9838y85W4P9vWnB3spfxL8sNAJ3BJZr6/xSVVTkR8ATiP4gpGPcC7gX8HLgOOpria3Msyc2LnN00iIp4CXAP8mD2fN/4pxefi7tP9EBGPpeho1UnRKLksM98XEcdTdGg9FLgJeFVm1lpXafVExHnA2zPzhe7P9jVnQ1ySpLlurp5OlyRpzjPEJUmqKENckqSKMsQlSaooQ1ySpIoyxCXtt4i4KiJOn+FtroiI36mbPm/8alqSHsoQl9SQKMzGe8YK4Hf2uZYkQ1xqdxHx7xFxY3m97Avr5vdHxPvLa2lfGxGHl/NfGhE/KedfXc77j3JgFMrrRL+rvP2+iHhDefsPI+L6iLil7rrcx0bEuoj4F+AnPHQ444l1Picivh8RP4yIfy3HiCcifhER7y3n/zgiTi3nryqvn35rRHwyIu6KiJXAXwMnRMTNEfG35eaXRcSXIuKOiPhcOfqdNO8Z4lL7e11mPgk4Hfj9iDisnL8UuLa8lvbVwBvK+e8CnlvOf1E57xrgqRFxMMV1zc8t5z8VuDoingOcRHEBkccDT4qIp5XrnAT8Y2Y+KjPvmqzAMnzfCTwrM58I3AC8tW6V+8v5HwPeXs57N8Wwno+iuMzl0eX8dwA/y8zHZ+YflvOeALwFOA04vq5+aV4zxKX29/sR8SPgWoqW8Enl/GFg/LPiG4Fjy9vfBT5VtrA7y3nXAE+jCL//oGjZLgGOy8x1wHPKn5uAH1JcCWz8ce7KzGv3UePZFAH73fKyoK8GjqlbPn6xl/o6n0IxlCeZ+Z/A9r1s/weZeU9mjgE3121Dmtfm6lXMpDmhHL/6WcA5mTkYEVdRXF0KYFfuGTd5lPL1nJm/HRFnAS8AboyIJ1FcFOh0YAPFNbdXUrTcxy85GcAHMvPjEx7/WGCgkVKBb2bmBVMsHx9ne3ed01Q/Tvf+bkOac2yJS+3tYGB7GeCnUrR49yoiTsjM6zLzXcBWYG1mDgMbgZcC36domb+d4jQ8wDeA19V9jn1URKyeRp3XAudGxInl/ZdGxMn7uM93gZeV6z8HOKSc3wcsn8ZjS/OWIS61t/8EuiLidooOX/s6rQ3wt2UHsp8A3wN+VM6/BtiSmTvL22vK/8nM/wI+D3w/In5M8Rl1w0GamVuB1wBfiIhbKP5QOHUfd3sv8JyyzpcCm4G+zHyA4rT8T+o6tkmahFcxk9QSEbEQGM3MkYg4B/hYZj6+1XVJVeLnSpJa5WjgsvK758Ps6V0vqUG2xCVJqig/E5ckqaIMcUmSKsoQlySpogxxSZIqyhCXJKmiDHFJkirq/wffnoqOJ+TJ/wAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "filtered_dist = {k: dist[k] for k in sorted_keys if k < 50}\n",
        "plt.plot(list(filtered_dist.keys()), list(filtered_dist.values()))\n",
        "plt.xticks([i for i in range(0, max(filtered_dist.keys()), 10)])\n",
        "plt.xlabel('answer length')\n",
        "plt.ylabel('Dataset percentage')\n",
        "plt.title('Percentage of the samples with answer length less than a certain value.');"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "sIuJIaW9FWtd"
      },
      "source": [
        "Even 10 seems to cover almost the totality of the dataset, let's see:"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "kmwuuMrIFP_Z",
        "outputId": "e7226026-6da3-4920-c17d-485267de338f"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.9720008389261745"
            ]
          },
          "metadata": {},
          "execution_count": 13
        }
      ],
      "source": [
        "dist[10]"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "ugougiSSFg83"
      },
      "source": [
        "10 covers 97% of the answers of the dataset. We are going to use 10 as our output length"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 14,
      "metadata": {
        "id": "o6YyXJflFp33"
      },
      "outputs": [],
      "source": [
        "out_len = 10"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "OVSf0uv5ITJ2"
      },
      "source": [
        "Lastly we'll convert our dataframes to huggingface datasets."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 15,
      "metadata": {
        "id": "hY1U1kQzVIsz"
      },
      "outputs": [],
      "source": [
        "ds_train = datasets.Dataset.from_pandas(train_df)\n",
        "ds_train = ds_train.remove_columns(set(ds_train.column_names)-set(['story', 'question', 'answer', 'span_start', 'span_end']))\n",
        "ds_val = datasets.Dataset.from_pandas(val_df)\n",
        "ds_val = ds_val.remove_columns(set(ds_val.column_names)-set(['story', 'question', 'answer', 'span_start', 'span_end']))\n",
        "ds_test = datasets.Dataset.from_pandas(test_df)\n",
        "ds_test = ds_test.remove_columns(set(ds_test.column_names)-set(['story', 'question', 'answer', 'span_start', 'span_end']))"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "uAdAXBLS3lDd"
      },
      "source": [
        "### Question answering - bert_generation approach"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "z3wKMK7gcNGG"
      },
      "outputs": [],
      "source": [
        "class GenModel(tf.keras.Model):\n",
        "\n",
        "  def __init__(self, model_name, input_len, output_len, **kwargs):\n",
        "      super(GenModel, self).__init__(**kwargs)\n",
        "      self.model_name = model_name\n",
        "      self.input_len = input_len\n",
        "      self.output_len = output_len\n",
        "      self.tokenizer = AutoTokenizer.from_pretrained(self.model_name)\n",
        "      self.model = TFEncoderDecoderModel.from_encoder_decoder_pretrained(self.model_name, \n",
        "                                                                         self.model_name,\n",
        "                                                                         encoder_from_pt=True,\n",
        "                                                                         decoder_from_pt=True,\n",
        "                                                                         tie_encoder_decoder=True)\n",
        "      self.model.config.decoder_start_token_id = self.tokenizer.cls_token_id\n",
        "      self.model.config_eos_token_id = self.tokenizer.sep_token_id\n",
        "      self.model.config.pad_token_id = self.tokenizer.pad_token_id\n",
        "      self.model.config.vocab_size = self.model.config.encoder.vocab_size\n",
        "\n",
        "  def call(self, inputs, **kwargs):\n",
        "      loss = self.model(input_ids=inputs['input_ids'],\n",
        "                        attention_mask=inputs['input_attention_mask'],\n",
        "                        labels=inputs['labels'], training=True).loss\n",
        "      return loss\n",
        "\n",
        "  def generate(self, **kwargs):\n",
        "      return self.model.generate(decoder_start_token_id=self.model.config.decoder.pad_token_id,\n",
        "                                 max_length=self.output_len,\n",
        "                                  **kwargs)\n",
        "  \n",
        "  def evaluate(self, ds, batch_size=8, **kwargs):\n",
        "    f1_buffer = []\n",
        "    for sample in tqdm(ds.iter(batch_size=batch_size)):\n",
        "      actual_bsize = len(sample['input_ids'])\n",
        "      out = self.model.generate(input_ids=tf.reshape(tf.convert_to_tensor(sample['input_ids']), (actual_bsize, self.input_len)),\n",
        "                                **kwargs)\n",
        "      generated = self.tokenizer.batch_decode(out, skip_special_tokens=True)\n",
        "      preprocessed = [[self.tokenizer.pad_token_id if token == -100 else token for token in output] for output in sample['labels']]\n",
        "      true = self.tokenizer.batch_decode(preprocessed, skip_special_tokens=True)\n",
        "      for i, sentence in enumerate(generated):       \n",
        "        f1_buffer.append(squad.compute_f1(sentence, true[i]))\n",
        "    print(np.mean(f1_buffer))\n",
        "\n",
        "  def prepare_dataset(self, ds):\n",
        "    print('Tokenizing...\\n')\n",
        "    ds = ds.map(self.tokenize_and_remap, batched=True, num_proc=4)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 492,
          "referenced_widgets": [
            "a68b3c6e4d2c4e8a8287a4e63291ff08",
            "1ee5c0f410e54bda8d725d14db2a227a",
            "b085728752cd4b1999dbab0eed63a3d7",
            "538c138c74c24739b1bfeeae1966c8ed",
            "18656f0b0870406c858e37adf5b5de8d",
            "361e3f9878b54b2fb37ab4416b0065cc",
            "ba4be997d44349dd8a8d0144855aabc3",
            "c4811485725f4effbb0c589e4e1de54f",
            "e3f294fcee164f9681b3a025d9c12de7",
            "20af1c404a284cf3827a86ad1b2efe88",
            "06f70bbfd5794c2da03877d47cb9ad59",
            "6a6c4695bdf34597a417a9ca02da986b",
            "56e58362f95443109bc80500efe14c72",
            "bc962701c4d1422784099213f1488f7f",
            "145809341cc24978aa85d42dcd0f851c",
            "b1ea85a8dab14fd6904131dfd83061f7",
            "35c0e742a3ab409c9738118738cae772",
            "495a134a858141b6accb59d932a46be0",
            "f5623efe1f7846f4aa317b6494bbb0e7",
            "1f314aa9005347608422df3d99e58ff9",
            "c68fd39fd5d6439c9ebcc81702a68012",
            "569e573164d445e8b11e7bb9034236e3",
            "d05a1653512b4de1aa331f967eca5999",
            "ac62dcc37eb74183b9dd96c1a6e685a1",
            "54d6109e768b43dabea23b288aecd0d9",
            "5fca857152f24325be0d9ad9b31edb65",
            "537cbf9f13db4aad8a176729cddd35e1",
            "fb278481596f4a2b95f8dfb7f32a30a3",
            "89955fa22feb4c6fa1b26e27c8767ad1",
            "e003729475824eb4a39d83c935b1de1b",
            "eeab9777039943b196895161d817f6ac",
            "f32d969478664ca9859a625bfd6d8a74",
            "7eed1ff2bcd041519bd68b6f3cbc91f7",
            "55baab91f6c049aaa461e8254d50d7c8",
            "954567760e1749baa73f0be0868ef69b",
            "d8ebaec53de3464d8f89eb8b3af92555",
            "c9b36d60d41141ebb6f1e777e5194dc7",
            "5a6c0f91e8de426495396f247bf7c053",
            "e2586e13500e438a9085bc53d27b68b8",
            "c1e4b88424a44f2b815f0d7b48a26570",
            "3b47033797024386800fd4e8af6668f8",
            "d5b409f3460c4f7e8a70273e5dd1fad9",
            "19e5356ca0f34e5f8f18329841c9f41a",
            "5e2e21a173dc416cb488ea23ed1333c3",
            "dc6190ef4e8f484cb0ec3f54d30b6793",
            "cb1f485b70ca46b49b3f6dadee4adb3e",
            "25ab2038b4064b5d87286688b9e10537",
            "984a839dfef84665be4a738826b4549d",
            "4b953f448bef4befbc98f4a465ad6c21",
            "a51b51de8dd84c828aa78f100a7629d3",
            "364fb9f621644346a126dfb4f80dd833",
            "aca44a6d27ac49fe8f53578f117b1e09",
            "e8daba08f0074044ba9cd722b579cf45",
            "635bc2ae55824a2881760f99b3315193",
            "4c601f8ed9894ddb952828a67a2cb565"
          ]
        },
        "id": "zoLHN9056L71",
        "outputId": "b843e9dd-5394-43e9-ff0c-b15317012c22"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/480 [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a68b3c6e4d2c4e8a8287a4e63291ff08"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/878k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "6a6c4695bdf34597a417a9ca02da986b"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/446k [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "d05a1653512b4de1aa331f967eca5999"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/1.29M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "55baab91f6c049aaa461e8254d50d7c8"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "Downloading:   0%|          | 0.00/316M [00:00<?, ?B/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "dc6190ef4e8f484cb0ec3f54d30b6793"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.bias', 'lm_head.decoder.weight', 'lm_head.dense.weight', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the weights of TFRobertaModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
            "All model checkpoint layers were used when initializing TFRobertaModel.\n",
            "\n",
            "All the layers of TFRobertaModel were initialized from the model checkpoint at /tmp/tmpefvxemqa.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n",
            "All PyTorch model weights were used when initializing TFRobertaForCausalLM.\n",
            "\n",
            "Some weights or buffers of the TF 2.0 model TFRobertaForCausalLM were not initialized from the PyTorch model and are newly initialized: ['roberta.encoder.layer.0.crossattention.self.query.weight', 'roberta.encoder.layer.0.crossattention.self.query.bias', 'roberta.encoder.layer.0.crossattention.self.key.weight', 'roberta.encoder.layer.0.crossattention.self.key.bias', 'roberta.encoder.layer.0.crossattention.self.value.weight', 'roberta.encoder.layer.0.crossattention.self.value.bias', 'roberta.encoder.layer.0.crossattention.output.dense.weight', 'roberta.encoder.layer.0.crossattention.output.dense.bias', 'roberta.encoder.layer.0.crossattention.output.LayerNorm.weight', 'roberta.encoder.layer.0.crossattention.output.LayerNorm.bias', 'roberta.encoder.layer.1.crossattention.self.query.weight', 'roberta.encoder.layer.1.crossattention.self.query.bias', 'roberta.encoder.layer.1.crossattention.self.key.weight', 'roberta.encoder.layer.1.crossattention.self.key.bias', 'roberta.encoder.layer.1.crossattention.self.value.weight', 'roberta.encoder.layer.1.crossattention.self.value.bias', 'roberta.encoder.layer.1.crossattention.output.dense.weight', 'roberta.encoder.layer.1.crossattention.output.dense.bias', 'roberta.encoder.layer.1.crossattention.output.LayerNorm.weight', 'roberta.encoder.layer.1.crossattention.output.LayerNorm.bias', 'roberta.encoder.layer.2.crossattention.self.query.weight', 'roberta.encoder.layer.2.crossattention.self.query.bias', 'roberta.encoder.layer.2.crossattention.self.key.weight', 'roberta.encoder.layer.2.crossattention.self.key.bias', 'roberta.encoder.layer.2.crossattention.self.value.weight', 'roberta.encoder.layer.2.crossattention.self.value.bias', 'roberta.encoder.layer.2.crossattention.output.dense.weight', 'roberta.encoder.layer.2.crossattention.output.dense.bias', 'roberta.encoder.layer.2.crossattention.output.LayerNorm.weight', 'roberta.encoder.layer.2.crossattention.output.LayerNorm.bias', 'roberta.encoder.layer.3.crossattention.self.query.weight', 'roberta.encoder.layer.3.crossattention.self.query.bias', 'roberta.encoder.layer.3.crossattention.self.key.weight', 'roberta.encoder.layer.3.crossattention.self.key.bias', 'roberta.encoder.layer.3.crossattention.self.value.weight', 'roberta.encoder.layer.3.crossattention.self.value.bias', 'roberta.encoder.layer.3.crossattention.output.dense.weight', 'roberta.encoder.layer.3.crossattention.output.dense.bias', 'roberta.encoder.layer.3.crossattention.output.LayerNorm.weight', 'roberta.encoder.layer.3.crossattention.output.LayerNorm.bias', 'roberta.encoder.layer.4.crossattention.self.query.weight', 'roberta.encoder.layer.4.crossattention.self.query.bias', 'roberta.encoder.layer.4.crossattention.self.key.weight', 'roberta.encoder.layer.4.crossattention.self.key.bias', 'roberta.encoder.layer.4.crossattention.self.value.weight', 'roberta.encoder.layer.4.crossattention.self.value.bias', 'roberta.encoder.layer.4.crossattention.output.dense.weight', 'roberta.encoder.layer.4.crossattention.output.dense.bias', 'roberta.encoder.layer.4.crossattention.output.LayerNorm.weight', 'roberta.encoder.layer.4.crossattention.output.LayerNorm.bias', 'roberta.encoder.layer.5.crossattention.self.query.weight', 'roberta.encoder.layer.5.crossattention.self.query.bias', 'roberta.encoder.layer.5.crossattention.self.key.weight', 'roberta.encoder.layer.5.crossattention.self.key.bias', 'roberta.encoder.layer.5.crossattention.self.value.weight', 'roberta.encoder.layer.5.crossattention.self.value.bias', 'roberta.encoder.layer.5.crossattention.output.dense.weight', 'roberta.encoder.layer.5.crossattention.output.dense.bias', 'roberta.encoder.layer.5.crossattention.output.LayerNorm.weight', 'roberta.encoder.layer.5.crossattention.output.LayerNorm.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
            "All model checkpoint layers were used when initializing TFRobertaForCausalLM.\n",
            "\n",
            "All the layers of TFRobertaForCausalLM were initialized from the model checkpoint at /tmp/tmpabqkys_b.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaForCausalLM for predictions without further training.\n"
          ]
        }
      ],
      "source": [
        "m = GenModel('distilroberta-base', 512, 10)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "1jCm4mYDKLAx"
      },
      "outputs": [],
      "source": [
        "def tokenize_and_remap(val, tokenizer, input_len, out_len):\n",
        "  input = tokenizer(val['question'], val['story'], \n",
        "                          max_length=input_len, truncation='only_second', \n",
        "                          padding='max_length', \n",
        "                          return_offsets_mapping=True)\n",
        "  out = tokenizer(val['answer'], \n",
        "                        max_length=out_len, \n",
        "                        truncation=True, \n",
        "                        padding='max_length', \n",
        "                        return_offsets_mapping=True)\n",
        "  out = [[-100 if token == tokenizer.pad_token_id else token for token in seq] for seq in out['input_ids']]\n",
        "  start_pos = []\n",
        "  end_pos = []\n",
        "  for i in range(len(val['question'])):\n",
        "    start = None\n",
        "    end = None\n",
        "    it = 0\n",
        "    while it < len(input['input_ids'][i]):\n",
        "      if input.sequence_ids(i)[it] == 1:\n",
        "        if input['offset_mapping'][i][it][0] >= val['span_start'][i] and start is None:\n",
        "          start = it\n",
        "        if input['offset_mapping'][i][it][1] > val['span_end'][i] and end is None:\n",
        "          end = it\n",
        "      it += 1\n",
        "    start_pos.append(start)\n",
        "    end_pos.append(end)\n",
        "  return {'input_ids': input['input_ids'], \n",
        "          'input_attention_mask': input['attention_mask'],\n",
        "          'labels': out,\n",
        "          'start_positions': start_pos, \n",
        "          'end_positions': end_pos}"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "se4e4fV-Z1A7"
      },
      "outputs": [],
      "source": [
        "def prepare_dataset(ds, model, delete_unans=False):\n",
        "  tokenizer = model.tokenizer\n",
        "  input_len = model.input_len\n",
        "  out_len = model.output_len\n",
        "  print('Tokenizing...\\n')\n",
        "  ds = ds.map(lambda x: tokenize_and_remap(x, tokenizer, input_len, out_len), \n",
        "             batched=True, \n",
        "             num_proc=4, \n",
        "             remove_columns=ds_v.column_names)\n",
        "  if delete_unans:\n",
        "    start_len = len(ds)\n",
        "    print('\\nRemoving unanswerable questions...\\n')\n",
        "    ds = ds.filter(lambda x: False if x['start_positions'] is None or x['end_positions'] is None else x['start_positions'] < x['end_positions'],\n",
        "                   num_proc=4)\n",
        "    print(f'\\nRemoved {start_len-len(ds)} rows. ({((start_len-len(ds))/start_len)*100}% of the dataset.)')\n",
        "  return ds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 182
        },
        "id": "LOXa0AQHAUTu",
        "outputId": "a0a540d6-4d5a-4d3a-dc89-4f0f068f1860"
      },
      "outputs": [
        {
          "output_type": "error",
          "ename": "NameError",
          "evalue": "ignored",
          "traceback": [
            "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
            "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
            "\u001b[0;32m<ipython-input-14-f9f04a6cbef0>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mtk_train\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprepare_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds_t\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdelete_unans\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mTrue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0mtk_val\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mprepare_dataset\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mds_v\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mm\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
            "\u001b[0;31mNameError\u001b[0m: name 'ds_t' is not defined"
          ]
        }
      ],
      "source": [
        "tk_train = prepare_dataset(ds_t, m, delete_unans=True)\n",
        "tk_val = prepare_dataset(ds_v, m)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "h69vs8oab84q"
      },
      "outputs": [],
      "source": [
        "class QuestionAnsweringTrainer(object):\n",
        "\n",
        "    def __init__(self, keras_model):\n",
        "      self.keras_model = keras_model\n",
        "      self.optimizer = tf.keras.optimizers.Adam(learning_rate=5e-05)\n",
        "      self.input_len = keras_model.input_len\n",
        "      self.out_len = keras_model.output_len\n",
        "\n",
        "    @tf.function\n",
        "    def compute_loss(self, inputs):\n",
        "      loss = self.keras_model(inputs=inputs)\n",
        "      return tf.reduce_mean(loss)\n",
        "\n",
        "    @tf.function\n",
        "    def train_op(self, inputs):\n",
        "      with tf.GradientTape() as tape:\n",
        "          loss = self.compute_loss(inputs=inputs)\n",
        "\n",
        "      grads = tape.gradient(loss, self.keras_model.trainable_variables)\n",
        "      return loss, grads\n",
        "\n",
        "    @tf.function\n",
        "    def batch_fit(self, inputs):\n",
        "      loss, grads = self.train_op(inputs=inputs)\n",
        "      self.optimizer.apply_gradients(zip(grads, self.keras_model.trainable_variables))\n",
        "      return loss\n",
        "\n",
        "    def train(self, train_set, epochs=3, batch_size=16, val_data=None):\n",
        "      loss_history = []\n",
        "      for e in range(epochs):\n",
        "        epoch_loss = []\n",
        "        for batch in tqdm(train_set.iter(batch_size=batch_size)):\n",
        "          actual_bsize = len(batch['input_ids'])\n",
        "          inputs = tf.reshape(tf.convert_to_tensor(batch['input_ids']), (actual_bsize, self.input_len))\n",
        "          mask = tf.reshape(tf.convert_to_tensor(batch['input_attention_mask']), (actual_bsize, self.input_len))\n",
        "          out = tf.reshape(tf.convert_to_tensor(batch['labels']), (actual_bsize, self.out_len))\n",
        "          loss = self.batch_fit({'input_ids': inputs, 'input_attention_mask': mask, 'labels': out})\n",
        "          epoch_loss.append(float(loss))\n",
        "          loss_history.append(float(loss))\n",
        "        print(f'Epoch {e} loss: {np.mean(epoch_loss)}')\n",
        "      return loss_history"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "wSWZbe-iim54",
        "outputId": "3b1ce81d-3b9b-41ca-87f9-224b8ffe336a"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "0it [00:00, ?it/s]/usr/local/lib/python3.8/dist-packages/tensorflow/python/autograph/impl/api.py:371: FutureWarning: Version v4.17.0 introduces a better way to train encoder-decoder models by computing the loss inside the encoder-decoder framework rather than in the decoder itself. You may observe training discrepancies if fine-tuning a model trained with versions anterior to 4.17.0. The decoder_input_ids are now created based on the labels, no need to pass them yourself anymore.\n",
            "  return py_builtins.overload_of(f)(*args)\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_encoder_decoder_model/encoder/roberta/pooler/dense/kernel:0', 'tf_encoder_decoder_model/encoder/roberta/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "WARNING:tensorflow:Gradients do not exist for variables ['tf_encoder_decoder_model/encoder/roberta/pooler/dense/kernel:0', 'tf_encoder_decoder_model/encoder/roberta/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "5268it [1:44:14,  1.19s/it]WARNING:tensorflow:Gradients do not exist for variables ['tf_encoder_decoder_model/encoder/roberta/pooler/dense/kernel:0', 'tf_encoder_decoder_model/encoder/roberta/pooler/dense/bias:0'] when minimizing the loss. If you're using `model.compile()`, did you forget to provide a `loss`argument?\n",
            "5269it [1:44:24,  1.19s/it]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 0 loss: 3.9428357932073927\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "5269it [1:44:16,  1.19s/it]\n"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 1 loss: 2.8924252496631757\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "5269it [1:44:17,  1.19s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Epoch 2 loss: 2.176779519563654\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "trainer = QuestionAnsweringTrainer(m)\n",
        "\n",
        "history = trainer.train(tk_train)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 282
        },
        "id": "0bgimqrqA4Fo",
        "outputId": "e3e148d0-a2fe-4d5b-8ba1-545c59f5c3c5"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7fa4f26403d0>]"
            ]
          },
          "execution_count": 17,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXUAAAD4CAYAAAATpHZ6AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3gU1foH8O+bCoRAgIReQu890gWkXFH42fXaxXK5XrnqtV5EvXZFvTbsYMHe8QoiKKD0Zui9h04IPRASUs7vj53dbJndnd2d7O4s38/z8LA7M7v7MiTvnD1zzntEKQUiIooNcZEOgIiIzMOkTkQUQ5jUiYhiCJM6EVEMYVInIoohCeH8sPT0dJWZmRnOjyQisrzly5cfVkplGDk2rEk9MzMT2dnZ4fxIIiLLE5FdRo9l9wsRUQxhUiciiiFM6kREMYRJnYgohjCpExHFECZ1IqIYwqRORBRDLJHUf1y5F58vMTxMk4jonGWJpD5l1X58m70n0mEQEUU9SyR1IiIyxjJJnQs0ERH5Z4mkLiKRDoGIyBIskdSJiMgYyyR1Bfa/EBH5Y4mkzs4XIiJjLJHUiYjIGMskdY5+ISLyzxJJnYNfiIiMsURSJyIiYyyT1Nn9QkTkn0WSOvtfiIiMsEhSB0epExEZYImkzhulRETGWCKpExGRMZZJ6op3SomI/LJEUmfvCxGRMZZI6kREZAyTOhFRDPGb1EXkIxE5JCLrnLbVFJGZIrJV+7tGRQbJ0S9ERMYYaalPAjDMbdsYALOVUi0BzNaeExFRhPlN6kqpeQCOum2+FMAn2uNPAFxmclw6cVT0JxARWV+wfep1lFIHtMcHAdTxdqCIjBKRbBHJzsvLC+rDhONfiIgMCflGqbINIPfajlZKTVBKZSmlsjIyMkL9OCIi8iHYpJ4rIvUAQPv7kHkh6eMapURE/gWb1KcAuEV7fAuAn8wJRx9HvxARGWNkSONXABYDaC0ie0XkdgDjAAwVka0AhmjPiYgowhL8HaCUus7LrsEmx+InjnB+GhGRNVliRim7X4iIjLFEUiciImMsk9TZ+0JE5J8lkjonHxERGWOJpE5ERMZYJqlz5SMiIv+skdTZ+0JEZIg1kjp4o5SIyAhLJHU21ImIjLFEUiciImOsk9TZ/0JE5JclkrqwTgARkSGWSOpERGSMZZI6e1+IiPyzRFJn5wsRkTGWSOpERGSMZZI6ywQQEflniaTOwS9ERMZYIqkTEZExlknq7HwhIvLPEkmdvS9ERMZYIqkTEZExlknqHPxCROSfJZI6a78QERljiaRORETGWCapK45/ISLyyxJJnZ0vRETGWCKpExGRMSEldRG5T0TWi8g6EflKRCqZFZg7jn4hIvIv6KQuIg0A3AMgSynVAUA8gGvNCsz1wyrkXYmIYk6o3S8JACqLSAKAKgD2hx4SEREFK+ikrpTaB+C/AHYDOADghFLqN/fjRGSUiGSLSHZeXl7QgbL7hYjIv1C6X2oAuBRAUwD1AaSIyI3uxymlJiilspRSWRkZGcF9FvtfiIgMCaX7ZQiAnUqpPKVUMYDJAPqYExYREQUjlKS+G0AvEakitnn8gwFsNCcsV6wSQERkTCh96ksBfA9gBYC12ntNMCkuIiIKQkIoL1ZKPQHgCZNi8fdZ4fgYIiJLs8SMUva+EBEZY4mkTkRExlgmqbPzhYjIP0skdY5+ISIyxhJJnYiIjLFMUufgFyIi/yyR1I+eLsbBk4WRDoOIKOpZIqnP2pgb6RCIiCzBEkmdiIiMsVRSLy1jxzoRkS+WSurLdx2LdAhERFHNUkmdiIh8Y1InIoohlkrqJWVlkQ6BiCiqWSqpf/vnnkiHQEQU1SyV1FftOR7pEIiIopqlknrOkYJIh0BEFNUsldSJiMg3JnUiohhiuaReUsoRMERE3lguqd/79apIh0BEFLUsl9SnrT0Q6RCIiKKW5ZI6ERF5Z8mkfqqoBKeKSlBaplBwtiTS4RARRY2ESAcQjA5P/AoAuKRzfUxZvR8544ZHOCIiouhgyZa63ZTV+wEAZ86WRjgSIqLoYOmkbtf2PzMiHQIRUVSIiaROREQ2TOpERDEkpKQuImki8r2IbBKRjSLS26zAAnW6qATf/LkbSnEdUyI6d4U6+uUNADOUUleJSBKAKibEFJT22oiYapUScVHHepEKg4goooJuqYtIdQD9AXwIAEqps0qpiBc8zy8qwXPTNmD0lysiHQoRUdiF0v3SFEAegI9FZKWIfCAiKe4HicgoEckWkey8vLwQPs4gBUycvxPT1hxAUUn5UMedh09jOksMEFGMCyWpJwDoBuBdpVRXAKcBjHE/SCk1QSmVpZTKysjICOHjjCl16lNv/Vj5UMfBr8zBP75YwSqPRBTTQknqewHsVUot1Z5/D1uSj6hHJq91ea6UwnfZe1Cm5fpRny3Hhwt2YsY6/Vb7pIU7secoV1giImsKOqkrpQ4C2CMirbVNgwFsMCUqEz343Ro89P0ax/PfNx3CMz9vwJ2fe/a5nzhTjCenbsANHyz12EdEZAWhjn65G8AX2siXHQBuDT0kc/2wYq/hY+3DIY8XnK2ocIiIKlRISV0ptQpAlkmxRJy9i0YBeG3mFnRqWB2D29aJaExERIGwZJVGsz05ZT0mLcrB3/s3AwDkF5bgjdlbAYAVIInIUlgmAMCkRTkAgE0H8yMbCBFRiM75pH62pHyIo0gEAyEiMsE53f3y2eIctK5bzfE8nlmdiCzunG6pP/7TepeWelwckzoRWds53VIHgJEfL3M8XrXHe+maP3OOYteRAlRNjsewDvWwPe8UduSdxtB2HB1DRNHjnG6pA0BJWXlZgbz8It1jDuUX4ur3FuPB71Y7Ji0NfmUu/vZptt/3P1VUgq+X+S8JvHzXUbR5fDqOnuYYeSIK3jmf1P35etluHM53TbRvasMdjXjip/UYM3ktlu486vO4d+fsQGFxGbJzXI8rOFuCTQdPGg+YiM5pTOp+jJm8Fk9NXe+y7ZWZWxyP35y9Fc9N814d4chpW+vf/+LY+i35aycswbDX5yP3ZKGxgDUfL9yJV53iJKJzA5O6AYe8dMsAtgQ/cf5Or/vtt17LtO6XM2dLdbtj7E/FbQTOmr0nAABfLt0dUMxPTd2A8QF8oyCi2MCkbsDOw6cDOv7giUIUFtta5vYkbU/aL0zfiDGT12LOFtfa8vYUb/b4m9yThZgwbzuX+SM6RzCpV4BeL8zGqM+WAyhP0vaUar8ZW1DkvTvmVFGJabGM/mIFnv9lE7YdOmXaexJR9GJSN0lZmXJZgGOe1hK396a4t5Td5znZ93+8aCc6PPEr9h8/47J/4vwdHp+plMLU1ft9LvyRX2i7QJSypU50TrBEUk9OiP4wx/64Fi0enY4/Nh9ybDtZWOyo/FhapvDRgp0oKtFPwPaUu3DbEQDA3mOuSb1A50br1DUHcPdXK/H+PM+ET0TnJktMPnrtr11w1xfRvZD013/uAQDc+vGfjm2dnvzN8fjb7D34Y3N5P7p733kwDemjp2xdOYGOjCGi2BX9TWAAlRPjIx1CyOzdIHoWbz+CkjLXFryRG5vuN2GDceDEGfy0al/wb0BEUcUSST0WZO865vLc3qe+YvcxXDdxiaPbxe6vE5Zg0kLvQyWd30M5jXHPPVmIzDHT/MazYf9JnDhTjOsnLsW9X6/CmbOlKCktw5NT1uMQW/5ElmWJ7pdYdtjHGPgnp+pPaiouLUPLR6ejY4PqAFxb6tvzjI1yuXj8fDSuWcUxGkdBYf62I5i0KAd7jxXgg1vO031dfmExPlmUg38MbIF4FkAjijqWSOrKy2xLK1u37yQqJcY7hj4asedoAYpLy3D4lK1swdp9tolJU1bvR1qVRGTWSnFZZBsANud6Lvyx91gBAGD30QJUSiz/svanVsrAuR6Ou+d/2Yivlu1B84yquKhjPcOxE1F4WCKpx6K3/tgW8GvOf+kP3e35hSV4+4/t6NCgmu5+ZzPWHUDeqfJaNoXFtr780jKFd+ZsB6DfR6+UQmFxGU5p4+vP+hhGSUSRY4mkziHW5rFXmXT3w/K9jscFZ0uwaPthLNlxFPcPbQUAaPrIL7a/01MAAB8tzMGlXRpUcLREFChLJHUyRnwUGRj2+nxMulW/nxwAzhSXt7z/zDmG6ycuBQCMn70VXRunOfbZSyas9lF7nogixxKjX9hSN2azn4WzP9EW2Nbz4oxNXvet3K2fwM0YCjlj3UGcLCwO+X2IyMYSSZ2M8dfP7Tz5yQz3fr0qpNfnHD6NOz9fjvu/We3zuMLiUhw+5X2UUEVatec49rmVbDDDtDUHAi4UR2SEJZJ6i9pVIx0CBWjl7mMo8zGKBgBOn7VNyLKPxvHm2glLkPXsLNNiC8Rlby9E33G/6+4r9fPv82X0lysw6JU5Qb+eyBtLJPXM9BRsePpCPDa8baRDIS/enL3VUQny1/UHcfk7i1xq0rw0YxMufmO+y2u81ZB3523t2G2H8l1q7YRi0bbDPitZ7j5S4NJN9P3yvWg+9hfsOer7guQLuxWpIlgiqQNAlaQE3N6vaaTDIB0nzhTjlZlbcO2EJThRUIy/a2Pv35+3HTd9uBTjpm/CO3O2Y8MB/WX5gp3CNOTVeS61dkJx/QdLMeTVuV7393/5D1zy5gLH86mr9wMAthmc7EUULpYa/eKvRUfh99niHEcL9/CpItzz9UrHvuMFxZi/9TDmbz0c8Pvaa99E0/95zpHyVnlFNLJ/WL4Xq/YcxzOXdaiAd6dzhWVa6hSdHv9pPT5ZvMvxfO4W3zdjM8dMw4kCWzeGfZFtvQR55buLHGPjnRWVlGLoq3MN1bepSI56+Sa+5wPfrcZnS3a5bFNKYZmfRcuJnIWc1EUkXkRWisjPZgREsW+PdmPUXttm44GTyD1ZiAEv/4EfV9omQa3wMozy/bk7sNWt7ztzzDQ8+/MGlJUpvDl7a4WPlCkq8beIuHk+X7IL17y/GL+uP6i7/8zZUpSWKbw7Zzt+83IMnVvM6H65F8BGAP7nqBMBmLP5kMeSfT2fnw0AeOi7Nbi8a0Pd1+UXFuPVmVt0932wYCfi4wTvz9uB5buPYdKtPVz2PzllPZIT4rDt0Cm8fUM3FJwtReXEeFROCryss/NCJxXdPbQ9zzbs0X3RFLu2/5mB4Z3qYdqaAwCAnHHDQ/7M/MJinDlbitrVKoX8XlZSWFyKvPwiNKpZJdKhhCSkpC4iDQEMB/AcgPtNiYhi3n9/00/M/pSU+u7JPq516xQUlWLdvhNoV8/WzoiLE0xymni1dOdR3PLRMrSuk4pf7+sfVCyhmLHOeIva1zXDXmHTntDNMviVuTiUX2TKBaIi5eUXYenOIxjRqb4p73f3Vysxc0Mutj9/saUrkIba/fI6gIcBeJ31IiKjRCRbRLLz8kKf/JJeNSnk96Do5Stt3/3VSh97yy3LOYoRby5As7G/4JK3F3jst6/pujk3H0dOFeH+bzwnUV0/cQn6vDDb72c5D4NUSqHzU7/hc7d+cWdbcvNx5+e+K3Pe/dVKj0VSfli+12Pc/xXvLvQbXzAO+SgHHU1unbQM//xyJY4XnPV/sAG/b7INjy2z+FjToJO6iIwAcEgp5fMnVCk1QSmVpZTKysjICPbjHNKqMKnHMvcJPf1eLJ/4s2Bb4KNo1u07iTluY9lfmF5eEqH7s7MweWV5uYMvl+4GACzafgT7T3guFjJ7Yy6WbC9f0OSZn233BTLHTMN/f9uME2eK8dj/1nm8bu3eE3hnzjb85bV5fmOeunq/o6yyvZ7PhgMnHUsm2u05av5MVyvZf9z2/xPKJLBYFEr3S18Al4jIxQAqAagmIp8rpW40JzQi733Jeoq9lEkY6TaW3dcko7E/rkXfFrW87l+15zg+XezaEj+Ub0sub/+x3eP4Kav34x4f3zCUUrr98vZ+e+ddxwrO4o1ZW9G5UXUMbF3b63ueK4ws+RgI63a4uAo6qSulHgHwCACIyEAAD4YjocfKiSfzObe4QzHg5Tle9735u2cd/Pt0um9OF5Xgpg+XYmuu78lJM9YdxAVtamPxDtflDB/9cS225J5ymXBXWFzq+PzFjwzy+b5GHTxRiDrVkqNqPkCgzIo9Vtr7HKdOUWfY6/67KEJhdv6yLxzi7Kmp67Fi93HkF3lfcBwA8otKMPqLFR4zY7doF4NTTguWOxds6/2Cfj2aQGzJzUevF2bj44U5Ib9XRfh62W7bvIYz+lU89ZLwe3O3O+YQBMviXermJHWl1Byl1Agz3sufm3o3CcfHUARt8lNCOFRm/9LqdQN8m71X50hPD3+/BrM3ea9f8032Hq/7AnHHJ9keE7ZytCqRi7Yf0XtJxNkvNgdOGO+CGzd9E27+aFlQn2e/1lt9+UzLtdRv7p2JxPjyplbDGpUjGA1R+EZLFJ71P+mprEyhqKQUny/Z5XKPYdbGXADAt3/uwZ6jBXh66gbY7y/a95mhqKQUo79Ygd1Hgi90Zhfu5GrtVF7OUrVf9Mx+YABem7kV7831vElFFA5lYVqu9ZPF3odK2t066U9HqYYTZ4ox+oIWLvsf/qF8YfLkRN9tuh+W78WV3fUngnmzcNthTFt7AAVnS/Cx2wSw9+duR7XKibiuR+OA3tPXil62/eZyvkYfO30W/V78HZ/e3gPdm9R0bJ+9MRdpVRJdtkULy7XU3SUnxCMpwfL/DLKwaBrX7Fx75+VfN/s8drefssEPfOd78ZJAvTB9Ex6ZvNbw8f5Oq9HT/v3yvbjm/cV+j9O7OPyZcxSnz5bi3Tk7XLbf/kk2rnzX/3tGgiWzoft/ptlDm4gCURLF46RzTxZ6HcftPBP1srcXYtF2z3kAR5zq6OQXFuOh71Z7lHjQU1Km8OrMLYaO9cf9xvaeowWYuSHX6353D3632mdRtGOnz+JEQbGj+8V59rEVWTKpp1ay9RoNaGWbzHRV94aoVsnyPUlkUb7GvUdaz+dno/lYz2qX7lbtOe5YbNxZgVM//sR5O/Dd8r34aMFObM3Nx5NT1uPEmWLdAmrztx7G+Nlb8eJ072vf+uPtUnnx+Pn426fZQb1nSWkZnv9lo0vMXZ+Zic5P/+Z4Pk4n5lkbc7ElV/8G/o68Uxj58TIUFrve8ygrUzh62pzZroGwZFL/4R998NCFrfHBLVkAgCa1UrDmyQt1j61brVLAfXhE56rlu465PM85Ur6OqnOSHfraPExalIPOT/2GrGdnYcHWw7qrQK3YbXs/vXVey8qUz4qajpr6btvztWGewXxDn7slDxPm7cDjOrN+/fE2G/ipqRswZ3Oex1yD12dvRbdnZuLQSc+ZyRXJks3bZhlVPW4AedOwRmW8cEVHfLVsdwVHRWR97ksH6nUtTZy/w2PbjR/aWvkfjczSfd8bP/D8FvD6rC0Y//s2tK6TigMnzrg0zI6ePuuoUOmte+VkoWvXjns305IdRzxKFtuPKfZTHM4umI61/cfPICM1GbO0LqJD+UVhrXhpyaQeiOjt7SSKPlNWuc7K3XQgHweOF+L8lumO2az5hd77yXe5DWW0N6ad++ZfnbkF/xrcEjM32sbnb9a6Naau3o/7vlmFpy/tgA8WeF44vLGPjnnNrSzztROW+HiVPbmbO3TpREEx+oz7HTf2ilzvgCW7X8zw3OVcMozI3eq9J1yevzhjE8b+uNbwzcOntIVP7DYcOIlL31rg0qoeP3srlurcuHxxxiaUlCmM/XEtduSVd/v8uj4X6/ad8Djezl57Z62PY+zcSwoU6MwGDsSK3cdcGo72xcn/2JSHEm2s65ni8C2qAsRwUr+pl/7M03dv6IaPRmbhhp6cmUpkVCiJyf1CAegPA/XWRf7yr5sx4k3PEsp2Q7W+7kC+lds/K9QJTle8s8jluf2aoZRylHp4Y9bWkD4jUDHb/eKtH+6ijvXCGwhRDDjppf5KsM6WlmHjgZMu2/RupjqbtuYAhncK7ffXnhaKSsqQOWYaOjdKc+zTG/pp5F6s/T1/WL4XXRvXAACXss3uo2IqWsy21NMqJwIAWtau6vfY2qnJaJaeUtEhEVmW2fempq8NfLWm0V+uQH6h/sXlx5V7DZVRsLOvM7t6j/5auHYfLdjp8jxzzDS89btry9t+bn5ec8BRXz+SYral3qJOKr68oye6NbFdOd+7sRuOntb/gVj26BAAiPgK9UTRamEQC5T4YrTgmbvXvXRl3PeN6+xXveT63LQN6NHUViv/z5xjHvudHT5VhIMnCrEsx7Pv3305xv1+vmGEW8wmdQDo0yLd8XhYB3a7EAXLvv5rpH3o1nIO5LiJ83eidqqxoYXDx89H7kljy/pF2+SzmOp+WfH4UAxpW8fw8e/c0A1jL25TgRERUTQxWkvfaEKPRjHVUq+ZkuS38pyzi3nTlIgqWPYu3109Zouplroz6y7ORUQV5dlpGyPyucPHzzd9opM3MZfUr+zWAADQuWGanyOJiMJj/f6TyMsPT5dOTHW/AMCgNnWQM254pMMgIoqImGupm+W8zBqOx50bVnc8zqxVJRLhEJHFmb3guTdM6l7YiwR9M6oXfvpnP8f2y7sGtryXsy/v6Ik5Dw4MNTQiIq+Y1A1qoc1MNVorokpSvMe2Pi3SUatqkqlxEZE1+Ftr1SxM6jqWjR3skbwn39UHcx8aaHhdRPf/vmcus1WFTIjTP+WPDW8baJhERB6Y1HU4F7S3l+qsVikRTWql+Fxt5a6BzXW3v/bXzo6qkZWT4vH29d08jrm9X1P8Y2BzXN61QSih++RtAQMiih1M6k6+HtULE27q7vMYX2sMPzzMc3bqlH/2xWVdXBO1XqU5EcG/h7XBRR3qGgs2QE/+Xztc0Lo2hnPCFVFMY1J30qtZLfylve+kWiPFd5/41d1db6Q2y6jqUZjfl0CO9WX8dV1dno/s2xQigrdvKP+WcH7LdPeX+VWveviW5SKKJRz9EmF9tWJgdd3WFhzZJxMvXdUJA1tnuGzf/OwwAMBLV3XC9ucvDkuMOeOG4+WrOunua1ijst/XJyd43szVw2GcRNbBpO7FPYNaYuGYQWjsltDi4wTXZDVCnNtl154gRQTxcYLHRrRDnACVEgI7xb2b10LHBtX9H6jj41vPw7R7+mHmff0d20RsN3mdzX5gAADgwvauxc+S4j1jzRk3HHMeugALxwyyvZ+Xzx5zEQujEfkSrtIlTOpexMUJGqT5b+16c12PxtjxwnAk6CRKX6omJ2Dq3f109zmv0qInPSUZ7etXR8s6qY4foE4N09CtcQ2X45pnVMWmZ4bh6qxGeN/pHoI92dsZWWDEl1DOnx2/JRAFJuikLiKNROQPEdkgIutF5F4zA7OKlrWrIj6u4q/BN/ZqjO/v7I13tD7x9KrJIb1fpUTbNwvnbxyNarom0D7Naxl+vxSdcfkPXtjKZ5xxYuvO8uXKbsFP9iI6F4XSUi8B8IBSqh2AXgBGi0g7c8KKfvahjWMuahOWPvRnL+uIxPg4RxLu1tiz1d6yTnnLupq2nF+LDGOt7cFtars8/9/ovnhsRPl/p6+hnAAQHxeH391a+mVeitJ9fntPAECrOql48pL2qJ3qI/GH4YJJFBbRfqNUKXVAKbVCe5wPYCOAihtkHaVCvaN9dfeGAU08qlPNlgBb10112X5lt4aO1jdg62L58o6eeO7yDkHF1TQ9BYlOXUdVk22133o39z5ippnbBSQhXnTPT023EUT9Wnh/z3CMGKjlZ0QTkZWY0qcuIpkAugJYqrNvlIhki0h2Xl6eGR8XFfq1tI1+aVwztD7fl6/ujDvOb+ax/a3ru+LXf/X32N61cQ388I/euHdwSwC+FwTu0yLdJdEHwj2ZplVJwu8PDMDzV+hfJPSS7/CO9TwaJ+lVk5GulUq4QPt24Ovf4O0LwgVuo49C8co1ndGjaU3T3s+bz27v4fI8w8c3FIo9likTICJVAfwA4F9KqZPu+5VSE5RSWUqprIwM834RI+22vplYNnYwWtRO9X9wEEZ0qu/RGrfr3qSmxw1Ys1q09ha5nmYZVR2jfNwnSdkvbvY+8E4Nq+veJP7xrj6oXa0Slo0djAf/0hoA0N2xOLjviV/unKtnejPuio4Y1d920fQ1zNPbTd1QL9rOUislujz/eOR5+OpvvUx7fyIgxHrqIpIIW0L/Qik12ZyQrEFEXMoJBGrl40NRarCQTJMKHAHi7Vrg6xqx6j9DkZKcgFOFJUhJTsCW3Hx00IZhPnNZeyzYlod/a7Nr3S829puxzufuhp6NMaBVBhrVrIKPRmbhtknZ+FkbATRvq/63OxHBU5d2wGVvL/T570tJTsA1WY0wYd4Or8coAM9e1gE/rtznsv3fw9pg1sZc7D5a4PMz/KlfvRL2nyh0OacrHh+KmilJ2JEXXYsWk/WFMvpFAHwIYKNS6lXzQjo31EhJMjSC5dPbeuC7O3t73W+/YRpseYG62gzR9vWrAQCe+L92qJqcgCpJ3q/3aVWSkBgfhxopSUhKiHMkdACokpSApWOHOCZvGSEijmRvX+SkQ4PqLu/rz829mzi6pJx5u2y+cEVHxzcEwJb8nWvoA0CXRmm4c4Ctns+ysYMNxdGuXjWPbTW17ibnC5z9vkLT9BQ8PKy1z/dM8HKzmEXgSE8o3S99AdwEYJCIrNL+hGcq5Tmkf6sM1E71/o2gRe1U5IwbjsFt63g9xpcODarj57v74d4hrQAAV2c1wrqnLjRtmGanClpW0Pmi0blhdTx9aQfcN7SV7rH2m8vOwycHt63t0dXk3ueZnBiHoe1sFxm9b2UjdGr4XJ3lOQTTPgJJr2tLRHDXwBa6cQNA9cqJuEynyNtjw9vipt5NvL6Ook/UlwlQSi1QSolSqpNSqov25xczg6Pw6NCgeoWNtX/j2i743+i+Qb/e282l2/pm+h1mCQCdGlRHaqVE5Iwbjtv6NkWVpHg8d3kHnxdKo966vht+dJutO7JPJnY4DXEd1r4unr+iIz69rYfH6CBv7h7kmuTd/5k544bjjvObGSrz8MUdPQ19JlW8jQc8bjlWCM4opQpVJSkBXfzMhA2GS+EzL02g9vWrITM9xfE8Lk6w4elhuKGnrYX79wG2G6ihLIDIu04AAAyASURBVFLe1W22roi4jK1/+4ZuqJKUgP6tfA8SmH7v+Y4uGfdupJsNtMiXeuke6tsiHR/ewpLL0eDM2dKwfA6TOoVNegirPvVvlYFFWv0ZO/uCI1WTXVuslRKN/Vj3aZ6OnHHDHck02e117i1kb33bdnrfSIx+A2pbrxpWPD4UOeNcS0sopdC5URqm/FP/286QtrZhoXV83LQf5DaxTI9eOWiyJiZ1CosljwzG7AcGBvw6e9GxMcPaoL7bsMMODarh4WGt8dpfuzi2bXpmGJY+MgQini1ef/41RL9P3s5eqti9QmdivOCKbg1cvpG8f1N3vHW9a/ljwHjp4vfd6vp3apiGS7vUx/U9G7tsn3hzlkt3jx5/5Zxfubozxl/rGSuZy6yy2v6ENKSRyKi6QdZhb5ZRFTnjhuvu07vJWCkxHpUS47HzBf3X+NK9SQ28c0M3vDFrKzbn5nvsf+v6bticm4/ODdNQVFL+VXrrc55J9UIvdfln3T8AZ0u81E9wYp9l28ZpNM0bOolXpHzW7u39muLDBTt13++/V3fGg9+t1t3Xum5qWOoXnevCdYrZUidycnHHeqiXZrsAJca7/hamJCegW+MaiI8Tn0M+fUlJTvC70Ir9uK9H9cLEm433hz8+oh3+r3N9x/PezcoLsl3ltHhL9mNDXF7nr/b+U5e0d3nuPvSToguTOlmOe/Exs716TRc8PqJd0HXtzdKrWS1Ur5zo/0An9vkGo/o3w1ej9Gerus+PSKtiu8hc2a2hy32PyonxGNqujsdFyOji66HqZGDGsJVE/ZBGokjY8uxFmBBA6zUYNVOScHu/pmHrAzWT0Yj1Fj9/5ZrOyH7MNtO1fvVK2PjMMEy8OQsjOtbDuCs6Oo4zo6tmzZN/8XvMTyEMhY1GNVPCU+uHSZ0sJSkhjv2/PgxtZ7uxfGmX+h77Ph55nqP8gq/RLssfG+JY6QqwDQW9tkf5Ddqrsxp5fe3f+5cXp6uUGOf1fki1SvrfQJzX+I3ERXXW/Z5F9MySFuC3rmAxqRPFEPuN5fb1PbsuLmhT21DpBdvNV8+EmjNuOFb/5y8u/fMewzidpw9oT2be199lIXT7vYpPbrNVrezqtDZAmHp2vKqoAn1A+P5tHP1CRIZVr2JrbU67px8S4uLQqo73WbL2L1Qt66SiZZ1UdGpQHdm7jmGANhFrQKsM5IwbjtNFJej9wmycLCyBUsD467qimdOkMXeT7+qDbo1rIHPMNFP+TeOu6Igxk9fiim6xsRwEW+pEFLD29aujdd1UiAjqOk18ci7r4D6MNTM9BVd1b+hRRz4lOQHz/z0InRul4e5BLXBJ5/o+v1HY19zt0MCzeJq75y/viGn3eK75++XfyssntNDW4q3oG+NGylqYgUmdiEJSpiWrT2/rgTStJX9eZg18GUCt+OqVE/HT6L4uZR0AePTJO88qvnew78liANC3RS20r18dX4/qhU+17p7/je6LPk4reGVl1sRv9/X3u16uEfYZvrf1beqxj90vRFShMlKTkZdfFPL72JNVm7qp6N28FlKSE3B9j8am3dBukFYZ+46fAeA6LNBeQVOvG6ZRzcrYc/SM43kvbcy+txu3reqU96XfOaA53pu73WdMm58dBqWANo/PcNn+xrVdselgPro3qYGPFrpOBAvXUFC21InOUb/cc74pwwbt3QoigsT4ONzUq4mpI5QWjhnkqDmfVtn7xK25Dw0EYFutKpSl48Zc1MZRgM1bobTkBNvMZXvrvnZqMro0SkNKcoJLnf5IYEud6ByVkZpsyjqpLWun4vCpI0hKqLg24l0DW3itO//ilR2RWikRtbRJVdf3bIyEOMGz0zY6tunpkVkT27ysPPXJrecB8D+s8rHhbTGgdQYuaO19QtymZ4bhVFFJ2IY0MqkTUUjeu6k71u87EfDsV7P89bzyMfTO3St6C7o7+9bHimJ6ybxl7apoUbsq5mwuX2IxIT7OZ0IHyusRhQuTOhGFpHrlRPQJYPlCq7m5dxN8ungXqlVOxLsBLo4eCexTJyLy4ZGL2qJ9/WqWWROWLXUiIh8qJ8Vj2j3nB/y6l67shGYZ3idRVRQmdSKiCnDNed5r5FQkdr8QEcUQJnUiohjCpE5EFEOY1ImIYgiTOhFRDGFSJyKKIUzqREQxhEmdiCiGSLhW4wAAEckDsCvIl6cDOGxiOGaK1tiiNS4gemOL1riA6I0tWuMCoje2QONqopTKMHJgWJN6KEQkWymVFek49ERrbNEaFxC9sUVrXED0xhatcQHRG1tFxsXuFyKiGMKkTkQUQ6yU1CdEOgAfojW2aI0LiN7YojUuIHpji9a4gOiNrcLiskyfOhER+WelljoREfnBpE5EFEMskdRFZJiIbBaRbSIyJgyf10hE/hCRDSKyXkTu1bbXFJGZIrJV+7uGtl1EZLwW3xoR6eb0Xrdox28VkVtMii9eRFaKyM/a86YislT7/G9EJEnbnqw936btz3R6j0e07ZtF5EKT4koTke9FZJOIbBSR3tFwzkTkPu3/cZ2IfCUilSJ1zkTkIxE5JCLrnLaZdo5EpLuIrNVeM170VlA2HtfL2v/lGhH5UUTS/J0Lb7+r3s53sLE57XtARJSIpIf7nPmKTUTu1s7dehF5yWl7xZ83pVRU/wEQD2A7gGYAkgCsBtCugj+zHoBu2uNUAFsAtAPwEoAx2vYxAF7UHl8MYDoAAdALwFJte00AO7S/a2iPa5gQ3/0AvgTws/b8WwDXao/fA/AP7fFdAN7THl8L4BvtcTvtPCYDaKqd33gT4voEwB3a4yQAaZE+ZwAaANgJoLLTuRoZqXMGoD+AbgDWOW0z7RwBWKYdK9prLwohrr8ASNAev+gUl+65gI/fVW/nO9jYtO2NAPwK24TG9HCfMx/n7QIAswAka89rh/O8VVhiNOsPgN4AfnV6/giAR8Icw08AhgLYDKCetq0egM3a4/cBXOd0/GZt/3UA3nfa7nJckLE0BDAbwCAAP2s/iIedfvkc50v7ge+tPU7QjhP3c+h8XAhxVYcteYrb9oieM9iS+h7tlzlBO2cXRvKcAch0SwKmnCNt3yan7S7HBRqX277LAXyhPdY9F/Dyu+rrZzSU2AB8D6AzgByUJ/WwnjMv/5/fAhiic1xYzpsVul/sv5R2e7VtYaF9/e4KYCmAOkqpA9qugwDqaI+9xVgRsb8O4GEAZdrzWgCOK6VKdD7D8fna/hPa8RURV1MAeQA+FlvX0AcikoIInzOl1D4A/wWwG8AB2M7BckTHObMz6xw10B5XRIy3wdaKDSYuXz+jQRGRSwHsU0qtdtsVDeesFYDztW6TuSJyXpCxBXXerJDUI0ZEqgL4AcC/lFInnfcp26UzrONBRWQEgENKqeXh/FyDEmD7GvquUqorgNOwdSU4ROic1QBwKWwXnfoAUgAMC2cMgYjEOfJHRB4FUALgi0jHAgAiUgXAWAD/iXQsXiTA9s2wF4CHAHwbSD99qKyQ1PfB1ndm11DbVqFEJBG2hP6FUmqytjlXROpp++sBOOQnRrNj7wvgEhHJAfA1bF0wbwBIE5EEnc9wfL62vzqAIxUQF2BrRexVSi3Vnn8PW5KP9DkbAmCnUipPKVUMYDJs5zEazpmdWedon/bYtBhFZCSAEQBu0C44wcR1BN7PdzCaw3aRXq39LjQEsEJE6gYRm+nnDLbfhcnKZhls36rTg4gtuPMWTJ9gOP/AdtXbAdt/ov0mQvsK/kwB8CmA1922vwzXG1ovaY+Hw/XmzDJte03Y+plraH92AqhpUowDUX6j9Du43ky5S3s8Gq43/b7VHreH6w2bHTDnRul8AK21x09q5yui5wxATwDrAVTRPusTAHdH8pzBsw/WtHMEz5t+F4cQ1zAAGwBkuB2ney7g43fV2/kONja3fTko71MP6znzct7uBPC09rgVbF0rEq7zVmGJ0cw/sN3R3gLbHeJHw/B5/WD7CrwGwCrtz8Ww9XHNBrAVtrvb9h8KAfC2Ft9aAFlO73UbgG3an1tNjHEgypN6M+0Hc5v2Q2C/615Je75N29/M6fWPavFuRgB3+/3E1AVAtnbe/qf98kT8nAF4CsAmAOsAfKb9UkXknAH4Cra+/WLYWnS3m3mOAGRp/87tAN6C243rAOPaBltCsv8OvOfvXMDL76q38x1sbG77c1Ce1MN2znyctyQAn2vvuQLAoHCeN5YJICKKIVboUyciIoOY1ImIYgiTOhFRDGFSJyKKIUzqREQxhEmdiCiGMKkTEcWQ/wc3gyHfb2gVCgAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 432x288 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "plt.plot(history)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8FqyxtQIA9qE",
        "outputId": "4e6bd74b-5558-419d-8cc1-2131a876a17e"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2it [00:57, 28.89s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.044643069348951694\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "m.evaluate(tk_val.select(range(0, 30)), \n",
        "           batch_size=16,\n",
        "           repetition_penalty=3.,\n",
        "           min_length=1,\n",
        "           no_repeat_ngram_size=3,\n",
        "           early_stopping=True,\n",
        "           num_beams=6)\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "I0s0QDXFCcCv",
        "outputId": "528f50ae-4577-4ed0-8c0b-c674c5e6eeef"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "336it [3:33:55, 38.20s/it]"
          ]
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "0.10238357474046454\n"
          ]
        },
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "\n"
          ]
        }
      ],
      "source": [
        "ds = tk_val\n",
        "f1_buffer = []\n",
        "for sample in tqdm(ds.iter(batch_size=64)):\n",
        "  actual_bsize = len(sample['input_ids'])\n",
        "  out = m.generate(input_ids=tf.reshape(tf.convert_to_tensor(sample['input_ids']), \n",
        "                   (actual_bsize, m.input_len)),\n",
        "                    batch_size=actual_bsize,\n",
        "                    repetition_penalty=4.,\n",
        "                    min_length=1,\n",
        "                    no_repeat_ngram_size=5,\n",
        "                    early_stopping=True,\n",
        "                    top_k=10,\n",
        "                    length_penalty=1.5,\n",
        "                   num_beams=4,\n",
        "                    eos_token_id=m.tokenizer.eos_token_id,\n",
        "                    pad_token_id=m.tokenizer.pad_token_id,\n",
        "                    )\n",
        "  generated = m.tokenizer.batch_decode(out, skip_special_tokens=True)\n",
        "  preprocessed = [[m.tokenizer.pad_token_id if token == -100 else token for token in output] for output in sample['labels']]\n",
        "  true = m.tokenizer.batch_decode(preprocessed, skip_special_tokens=True)\n",
        "  for i, sentence in enumerate(generated):\n",
        "    # print(true[i] + ' | ' + sentence + ' | ' + str(out[i]))\n",
        "    # print()       \n",
        "    f1_buffer.append(squad.compute_f1(sentence, true[i]))\n",
        "print(np.mean(f1_buffer))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "jPm1xG9_AaLO"
      },
      "outputs": [],
      "source": [
        "m.model.save_pretrained('/content/drive/MyDrive')"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "SYueojD5RFR5"
      },
      "source": [
        "## Question answering - seq2seq approach\n",
        "\n",
        "Trainer, Encoder and Decoder classes are based on [this implementation](https://gitlab.com/nlp-unibo/nlp-unibo-material/-/blob/master/2022-2023/Assignment%202/tf_seq2seq_bert.py) with little to no tweaks."
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 16,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6ernvGKHRdFW",
        "outputId": "a6232a3b-cec9-4106-e9b5-80abb5d18b2a"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: tensorflow_addons in /usr/local/lib/python3.8/dist-packages (0.19.0)\n",
            "Requirement already satisfied: typeguard>=2.7 in /usr/local/lib/python3.8/dist-packages (from tensorflow_addons) (2.7.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.8/dist-packages (from tensorflow_addons) (21.3)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.8/dist-packages (from packaging->tensorflow_addons) (3.0.9)\n"
          ]
        }
      ],
      "source": [
        "from transformers import TFAutoModel\n",
        "!pip install tensorflow_addons\n",
        "import tensorflow_addons as tfa\n",
        "tf.config.run_functions_eagerly(True)\n",
        "\n",
        "\n",
        "class Seq2SeqBertTrainer(object):\n",
        "\n",
        "    def __init__(self, encoder, decoder, max_length, out_tokenizer):\n",
        "        self.encoder = encoder\n",
        "        self.decoder = decoder\n",
        "        self.max_length = max_length\n",
        "        self.ce = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True, reduction='none')\n",
        "        self.optimizer = tf.keras.optimizers.Adam(learning_rate=1e-03)\n",
        "        self.out_tokenizer = out_tokenizer\n",
        "\n",
        "    @tf.function\n",
        "    def compute_loss(self, logits, target):\n",
        "        loss = self.ce(y_true=target, y_pred=logits)\n",
        "        mask = tf.logical_not(tf.math.equal(target, 0))\n",
        "        mask = tf.cast(mask, dtype=loss.dtype)\n",
        "        loss *= mask\n",
        "        return tf.reduce_mean(loss)\n",
        "\n",
        "    @tf.function\n",
        "    def train_op(self, inputs):\n",
        "        with tf.GradientTape() as tape:\n",
        "            encoder_output, encoder_h, encoder_s = self.encoder({'input_ids': inputs['encoder_input_ids'],\n",
        "                                                                 'attention_mask': inputs['encoder_attention_mask']})\n",
        "\n",
        "            decoder_input = inputs['decoder_target'][:, :-1]  # ignore <end>\n",
        "            real_target = inputs['decoder_target'][:, 1:]  # ignore <start>\n",
        "\n",
        "            self.decoder.attention.setup_memory(encoder_output)\n",
        "\n",
        "            decoder_initial_state = self.decoder.build_initial_state(self.decoder.batch_size, [encoder_h, encoder_s])\n",
        "            predicted = self.decoder({'input_ids': decoder_input,\n",
        "                                      'initial_state': decoder_initial_state}).rnn_output\n",
        "\n",
        "            loss = self.compute_loss(logits=predicted, target=real_target)\n",
        "\n",
        "        grads = tape.gradient(loss, self.encoder.trainable_variables + self.decoder.trainable_variables)\n",
        "        return loss, grads\n",
        "\n",
        "    @tf.function\n",
        "    def batch_fit(self, inputs):\n",
        "        loss, grads = self.train_op(inputs=inputs)\n",
        "        self.optimizer.apply_gradients(zip(grads, self.encoder.trainable_variables + self.decoder.trainable_variables))\n",
        "        return loss\n",
        "\n",
        "    # @tf.function\n",
        "    def generate(self, input_ids, attention_mask=None):\n",
        "        batch_size = input_ids.shape[0]\n",
        "        encoder_output, encoder_h, encoder_s = self.encoder({\n",
        "            'input_ids': input_ids,\n",
        "            'attention_mask': attention_mask\n",
        "        })\n",
        "\n",
        "        start_tokens = tf.fill([batch_size], self.out_tokenizer.word_index['<start>'])\n",
        "        end_token = self.out_tokenizer.word_index['<end>']\n",
        "\n",
        "        greedy_sampler = tfa.seq2seq.GreedyEmbeddingSampler()\n",
        "        decoder_instance = tfa.seq2seq.BasicDecoder(cell=self.decoder.wrapped_decoder_cell,\n",
        "                                                    sampler=greedy_sampler,\n",
        "                                                    output_layer=self.decoder.generation_dense,\n",
        "                                                    maximum_iterations=self.max_length)\n",
        "        self.decoder.attention.setup_memory(encoder_output)\n",
        "\n",
        "        decoder_initial_state = self.decoder.build_initial_state(batch_size, [encoder_h, encoder_s])\n",
        "        decoder_embedding_matrix = self.decoder.embedding.variables[0]\n",
        "        outputs, _, _ = decoder_instance(decoder_embedding_matrix,\n",
        "                                         start_tokens=start_tokens,\n",
        "                                         end_token=end_token,\n",
        "                                         initial_state=decoder_initial_state)\n",
        "        return outputs\n",
        "\n",
        "    def translate(self, generated):\n",
        "        return self.out_tokenizer.sequences_to_texts(generated.sample_id.numpy())\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 17,
      "metadata": {
        "id": "h4yVqtQSRtYf"
      },
      "outputs": [],
      "source": [
        "class Encoder(tf.keras.Model):\n",
        "\n",
        "    def __init__(self, model_name, decoder_units):\n",
        "        super(Encoder, self).__init__()\n",
        "        self.model = TFAutoModel.from_pretrained(model_name, from_pt=True)\n",
        "        self.reducer = tf.keras.layers.Dense(decoder_units)\n",
        "\n",
        "    def call(self, inputs, training=False, **kwargs):\n",
        "        model_output = self.model(inputs)\n",
        "        all_outputs = model_output[0]\n",
        "        pooled_output = model_output[1]\n",
        "        pooled_output = self.reducer(pooled_output)\n",
        "        return all_outputs, pooled_output, pooled_output\n",
        "\n",
        "\n",
        "class Decoder(tf.keras.Model):\n",
        "\n",
        "    def __init__(self, vocab_size, max_sequence_length, embedding_dim, decoder_units, batch_size):\n",
        "        super(Decoder, self).__init__()\n",
        "\n",
        "        self.max_sequence_length = max_sequence_length\n",
        "        self.batch_size = batch_size\n",
        "\n",
        "        self.decoder_units = decoder_units\n",
        "        self.embedding = tf.keras.layers.Embedding(input_dim=vocab_size,\n",
        "                                                   output_dim=embedding_dim)\n",
        "        self.decoder_lstm_cell = tf.keras.layers.LSTMCell(self.decoder_units)\n",
        "\n",
        "        self.attention = tfa.seq2seq.BahdanauAttention(units=self.decoder_units,\n",
        "                                                       memory=None,\n",
        "                                                       memory_sequence_length=self.batch_size * [max_sequence_length])\n",
        "\n",
        "        self.wrapped_decoder_cell = tfa.seq2seq.AttentionWrapper(self.decoder_lstm_cell,\n",
        "                                                                 self.attention,\n",
        "                                                                 attention_layer_size=self.decoder_units)\n",
        "\n",
        "        self.generation_dense = tf.keras.layers.Dense(vocab_size)\n",
        "        self.sampler = tfa.seq2seq.sampler.TrainingSampler()\n",
        "        self.decoder = tfa.seq2seq.BasicDecoder(self.wrapped_decoder_cell,\n",
        "                                                sampler=self.sampler,\n",
        "                                                output_layer=self.generation_dense)\n",
        "\n",
        "    def build_initial_state(self, batch_size, encoder_state):\n",
        "        initial_state = self.wrapped_decoder_cell.get_initial_state(batch_size=batch_size, dtype=tf.float32)\n",
        "        initial_state = initial_state.clone(cell_state=encoder_state)\n",
        "        return initial_state\n",
        "\n",
        "    def call(self, inputs, training=False, **kwargs):\n",
        "        input_ids = inputs['input_ids']\n",
        "        input_emb = self.embedding(input_ids)\n",
        "        decoder_output, _, _ = self.decoder(input_emb,\n",
        "                                            initial_state=inputs['initial_state'],\n",
        "                                            sequence_length=self.batch_size * [self.max_sequence_length-1])\n",
        "        return decoder_output\n"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "The first thing to do is to add the ```<start>``` and ```<end>``` tokens to the answers in the train and validation sets.\n",
        "\n"
      ],
      "metadata": {
        "id": "VBAzh3mIV1Ou"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 18,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 168,
          "referenced_widgets": [
            "ed35025355c843629080d12fad8b215e",
            "6b7d9855f305481fa64e7b96d3213221",
            "338f45f7bff04e76a617d132c94f9128",
            "ada1129519214046b9a2d7f7e8a7435d",
            "fbb36868517548d89d0384e0c6564d3d",
            "87edf2191c1247eab6136587cdcfb59d",
            "001b597009914c36aee4114cf93127bf",
            "c7700c4a6d8b4398959fd83607e4b28e",
            "b90309fc12264b669b9f7ff28fbf8716",
            "ecc7ace1d1614c559b6285a5c47269d7",
            "6c296c5134674ca998607abc2773ff26",
            "a51158f0209b448ea592916964f6e7fc",
            "380d5460dd2e4d8eaae19a9caa251954",
            "593a92109ee64ea99ca26fa9042e7a9d",
            "2892d2b28c3d40998b5127317038aadd",
            "95309f66bf664e54b7c2a177ad6b61b6",
            "94c90de8d86f47168f5c76cc75b44bd7",
            "e817dc63ea8f4d4dbb95317d03b5bbe3",
            "f36af37b73514f1184f115cd19f57c07",
            "61cc8aeb389d464b9f81dcf14d0edc95",
            "0bf16b30f9cf426ba668d8a466edbcb4",
            "ac7b23622d6f40278d5d3f830ed8050d"
          ]
        },
        "id": "lXY-OzjKymt8",
        "outputId": "7f6cb509-ffa2-4e8f-fa48-957c2ad09913"
      },
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/85824 [00:00<?, ?ex/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "ed35025355c843629080d12fad8b215e"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "  0%|          | 0/21452 [00:00<?, ?ex/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "a51158f0209b448ea592916964f6e7fc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "['<start> It was formally established in 1475 <end>',\n",
              " '<start> research <end>',\n",
              " '<start> history, and law <end>',\n",
              " '<start> philosophy, science and theology <end>',\n",
              " '<start> a  project <end>']"
            ]
          },
          "metadata": {},
          "execution_count": 18
        }
      ],
      "source": [
        "def add_start_end_pad(val):\n",
        "  return {'answer': '<start> ' + val['answer'] + ' <end>'}\n",
        "\n",
        "ds_train = ds_train.map(add_start_end_pad)\n",
        "ds_val = ds_val.map(add_start_end_pad)\n",
        "ds_train.select(range(0, 5))['answer']"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "Next we'll create the tokenizer that will be fit on the answers and we'll save its vocabulary size."
      ],
      "metadata": {
        "id": "V2C8VxFiWZLz"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 19,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "6aA_obko3Fu2",
        "outputId": "51793310-8f47-4142-b525-1f7a2d375db5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "44385\n"
          ]
        }
      ],
      "source": [
        "out_tokenizer = tf.keras.preprocessing.text.Tokenizer(filters='', oov_token='<UNK>')\n",
        "out_tokenizer.fit_on_texts(ds_train['answer'] + ds_val['answer'])\n",
        "vsize = len(out_tokenizer.word_index) + 1\n",
        "print(vsize)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We are now ready to create the encoder and decoder. Both of them will have 128 units and the decoder will work with an embedding size of 200. The chosen batch size is 8 for memory reasons."
      ],
      "metadata": {
        "id": "YZfowxNiW-7d"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 20,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ReuzItedMp79",
        "outputId": "f53ef1f8-9424-450c-9881-88014a5abcc5"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the PyTorch model were not used when initializing the TF 2.0 model TFRobertaModel: ['lm_head.bias', 'lm_head.layer_norm.weight', 'lm_head.dense.bias', 'lm_head.layer_norm.bias', 'lm_head.dense.weight', 'lm_head.decoder.weight']\n",
            "- This IS expected if you are initializing TFRobertaModel from a PyTorch model trained on another task or with another architecture (e.g. initializing a TFBertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing TFRobertaModel from a PyTorch model that you expect to be exactly identical (e.g. initializing a TFBertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "All the weights of TFRobertaModel were initialized from the PyTorch model.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFRobertaModel for predictions without further training.\n"
          ]
        }
      ],
      "source": [
        "bsize = 8\n",
        "encoder = Encoder('distilroberta-base', 128)\n",
        "decoder = Decoder(vsize, out_len, 200, 128, bsize)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "At this point our samples are still encoded by strings. The next step is to create a function that, for each sample:\n",
        "- Takes the question and its context and tokenizes them in a single vector of the form ```<start> QUESTION_TOKENIZATION <sep> CONTEXT_TOKENIZATION <end>```. This tokenization will be carried on by the bert tokenizer.\n",
        "- Takes the answer and tokenizes it in a single vector. This tokenization will be carried on by the ```out_tokenizer```\n",
        "- Remaps the ```span_start``` and ```span_end``` fields to be token based instead of character based. These fields encode where the answer is located in the context by its start character and ending character; but they lose their meaning if the answer is expressed in tokens instead of strings. The second part of the function carries on this remap, if the part of the context that contains the question got truncated, one or both fields will be set to ```None```.\n",
        "\n",
        "We developed this function to work on batches of data to take advantage of the paralleling processing offered by the Huggingface datasets."
      ],
      "metadata": {
        "id": "ymdGNViTXXbL"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 21,
      "metadata": {
        "id": "_dH36naqr_qe"
      },
      "outputs": [],
      "source": [
        "def tokenize_and_remap_s2s(val, in_tokenizer, out_tokenizer, input_len, out_len):\n",
        "  # Tokenizing input and output\n",
        "  input = in_tokenizer(val['question'], val['story'], \n",
        "                          max_length=input_len, truncation='only_second', \n",
        "                          padding='max_length', \n",
        "                          return_offsets_mapping=True)\n",
        "  tokenized = out_tokenizer.texts_to_sequences(val['answer'])\n",
        "  out = tf.keras.preprocessing.sequence.pad_sequences(tokenized,\n",
        "                                                      padding='post',\n",
        "                                                      maxlen=out_len)\n",
        "  # Remapping span start and end\n",
        "  start_pos = []\n",
        "  end_pos = []\n",
        "  for i in range(len(val['question'])):\n",
        "    start = None\n",
        "    end = None\n",
        "    it = 0\n",
        "    while it < len(input['input_ids'][i]):\n",
        "      if input.sequence_ids(i)[it] == 1:\n",
        "        if input['offset_mapping'][i][it][0] >= val['span_start'][i] and start is None:\n",
        "          start = it\n",
        "        if input['offset_mapping'][i][it][1] > val['span_end'][i] and end is None:\n",
        "          end = it\n",
        "      it += 1\n",
        "    start_pos.append(start)\n",
        "    end_pos.append(end)\n",
        "    \n",
        "  return {'input_ids': input['input_ids'], \n",
        "          'input_attention_mask': input['attention_mask'],\n",
        "          'labels': out,\n",
        "          'start_positions': start_pos, \n",
        "          'end_positions': end_pos}"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "```prepare_dataset``` wraps the previous function. If specified, this function will delete all the unanswerable questions (the ones with span_start or span_end set to None). We decided to remove this samples from the training set but we will keep the unanswerable question in the training and validation sets. The idea is that these two sets should test our model performances \"in the real world\", and in the real world it may happen that some questions have a context that is too long for our model to handle so it seems wrong to delete these samples from the evaluation sets."
      ],
      "metadata": {
        "id": "eX00GpopbSu6"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 22,
      "metadata": {
        "id": "Os0C_0HatrfC"
      },
      "outputs": [],
      "source": [
        "def prepare_dataset(ds, in_tokenizer, input_len, out_len, out_tokenizer, delete_unans=False):\n",
        "  print('Tokenizing...\\n')\n",
        "  ds = ds.map(lambda x: tokenize_and_remap_s2s(x, in_tokenizer, out_tokenizer, input_len, out_len), \n",
        "             batched=True, \n",
        "             num_proc=4, \n",
        "             remove_columns=ds_val.column_names)\n",
        "  if delete_unans:\n",
        "    start_len = len(ds)\n",
        "    print('\\nRemoving unanswerable questions...\\n')\n",
        "    ds = ds.filter(lambda x: False if x['start_positions'] is None or x['end_positions'] is None else x['start_positions'] < x['end_positions'],\n",
        "                   num_proc=4)\n",
        "    print(f'\\nRemoved {start_len-len(ds)} rows. ({((start_len-len(ds))/start_len)*100}% of the dataset.)')\n",
        "  return ds"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 23,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 765,
          "referenced_widgets": [
            "bb0114e630524f02953e7c415ba2fc3a",
            "6c1b789371084cc99831f8ee67decc1c",
            "a40d37987e064ea2bd773c35666fcaab",
            "1d081a07d0494faa8834b838d2c23566",
            "8fdc7b9298464e13bfbc323912737a8a",
            "338589e812e848198db439fee3dfa790",
            "db84f7a47d0a42a5ba602dc8fe602eb8",
            "3c1c2a45c3cc416f85ce48d6cb0758c3",
            "8b174ab0d1c743c98ccfa6ba422e72b6",
            "47aa4f2d3f324f1d8ce7ff9a8c2cf411",
            "23be94c3cd154d75b0b4e05077c27199",
            "88b1f3a9e23d4db9865d7a11cacebb50",
            "7bda13cfd4e541e392124813bce25e4f",
            "84372be77d5b4bceb832289eea82d600",
            "b4b817b41d6c499394a2e2628d8ce18a",
            "16a9ee1c222f480aa93b615cef08659c",
            "13b681c3c4ed4c75befef92c225de3a7",
            "64506eb279484e17872d1fe1c2b79c23",
            "913088feae2a4aa19cfd7cb87eae3c71",
            "68d51c9637454ad98de1e836b167bbdf",
            "974f022b4961402baede2ce2ed55917d",
            "796abc7ded4b405ba9dda5df9efddf39",
            "4b03fa2191a54d9f88d693eb2cd4dbe3",
            "505001abdbf641dd9eb590a5c908adf3",
            "da17f369ce54415e85f65021f7acc828",
            "cc01844e033949c6bab33e728c6e0f9e",
            "94050dfb9f0f41ea8df1b1f430dc8862",
            "f14be8d1068e4ae8913583891f63bb3c",
            "113820730fbb4bebabe2e5d20cef3d62",
            "5e51abebe07b41729fbfcaf9301b2a3b",
            "20ec9b04cb8a4193852cb0186748012f",
            "b752000f9fa94139bc11ed4c4870d1f8",
            "b34a78cecc7844d1b4d4a254cb00894e",
            "91ab0629fd7a47119a1629b18d687f21",
            "ccbe8b01447444ccb226d47d2e801d51",
            "41accc074902447b9155ce14847e1d00",
            "1644cf3859be470599bc47e804e191e6",
            "ed5df1bfbea849fe8d2ad0e7b370047e",
            "92c328913e6d43da9ef99a97f51f1f5d",
            "5b76e0933f0144faa48438a25d0e9477",
            "d500cb3e1a614cb1b56007ba36a63af3",
            "4f9d14e67dfa47cda198d4fbd5119884",
            "b7ffc9558988439bbca7099f8d9da531",
            "846146b0fe9d47d2bf795011f3438225",
            "1c25a2b2ccf845368b22a678a67a7cf1",
            "7533494a57624d3397d3b9919793c8c6",
            "2d748dc8d78a4b79bfc550eece411650",
            "1560a247acff4b8c81c0004274a74920",
            "ecf12d4c9e7f4a53aa14c7b9df2bc912",
            "0c0325a3635240afb6bd07a1e92d6854",
            "b2326eb1310149949aa6f23bf7bf41d7",
            "4462da40e82d4d6b8dbb4e69e199498a",
            "55d0fbe2f3bd42f7a22b8ee6b80a05c3",
            "52a32dbfcdb24921b135d7a397daaaf0",
            "33721ab988bb4d6a84aef7316c9aeb4f",
            "1faa01f2b1f248f6b941331e7a9228cc",
            "b2104de8d3a7410d984d9f72602e52f6",
            "56d6feed048d43b1b350ff7a69d4255e",
            "03d0a7a9428440eeb5d09f91ec0aa7a0",
            "1a8a7e75a96b4d2bab933a9131e0d4fe",
            "e9f79dff899a43f1bd1a8ec756439ebd",
            "8de80b1a0e10469dae7ef39f39c929fd",
            "1aa6c888c7324e4f9ed004ebe85183a3",
            "908111a195ef4142a3828faec82676c9",
            "1d486975f18b4aeab3ec11ad93619e2e",
            "00aee941958741c1af04ccfe630a9da6",
            "e160666b91b149ec960f7e0eca9135ab",
            "5e447b8656564f629866e7db4f5e148a",
            "07af587a84e74543977bdf2a85cec029",
            "821cf4378900472da5f2f3e270836668",
            "188fbc1c331048afae97ffa32fda9e44",
            "25e33db0d5b64408b358b855ff1f7dcd",
            "4643f9b89ef7406183f817843c80c40d",
            "2c450774bb5d4e70ad3a5b7f9fd6e707",
            "7a42ea74a2c74e0ea8b2b6bfc5e88003",
            "b370334a879f42c3829f6c5ae97a969d",
            "e0f40ee5ea33461290ba269be1684da8",
            "4e98f41b8f2a40f4bc1d65e842ca0b90",
            "3e15fb240e9847578096286fb9c1eb73",
            "9012ab4cd5c448b5be013f83a0f52a98",
            "21f550336293412780a8c8ba7cd90fd6",
            "562e115ae6f14897b97829a6f1f43f4e",
            "b2bd27bb7fea439e94471d464b5f1574",
            "4c31fb54a1cf4af5bd03b8485cbdfc29",
            "5a07ecd50ffa4192a4a24fe499559d96",
            "d80342cea6344ae5b0cd910d8100a500",
            "99b2f203217b43929c9c9ad049424b11",
            "54ccff485e7347c18e4e12522b2ded90",
            "5a0dadd77b5346909bda88dc8afe3ae1",
            "2c048fc882d54b638235417a1f122638",
            "b458917039134768a057b9e4dd93735e",
            "f946aa0f31c844bc8e78b788314e1e73",
            "50a5fd4fe2274f9388dbf0dfb5f384d9",
            "4786cdc6a7d644ba826a43b226c851ef",
            "e65c1356bdac466ea7cd223494d4114e",
            "7347da795d394cae91b452dc0ca58027",
            "6a0f660bcd99440dbf99002f64e790cc",
            "37c8e14302dc4cf39189ff9937b95366",
            "88cbe7373e4146c3b98e000edf22bfbb",
            "81bf3840fa6f48808e09e67870b1c39c",
            "b950fe152e094ccd8eb098dbe48f6766",
            "8c1ed1b009384f3998d3fa7071365d33",
            "6dadc0e8ab174a919bdf994602747df9",
            "4812fef0354a4cb6ae35d637418c57c7",
            "96966eb9e87949e2bf41d4253234f29a",
            "3b354d90adac4050aa98393968d8a8ab",
            "bda1f554b98245938111c0ecdfd5508b",
            "06a7cec6c0134983a8c9c273590f5920",
            "1ac2f62152f34a169c3240199d96f24e",
            "a84cfd4d64dc40a4860683175d109b03",
            "9ae329ef5d7b49d3a46a195cf9148f1f",
            "bba32c0c50684372962be5b3c4be3c2c",
            "a6ca682e5efa4338b54537f631c4696a",
            "fd247caa30c74b1daef472d59d29be36",
            "41f41df8445d4bd9bab92efeb3833796",
            "3c2aaed9c90b48c1be2162983906988e",
            "a461c15ab5784a7bb67073a21fe6ce0d",
            "2b3c7313fbb54fb5b0fae56669767a5d",
            "20feca53a7ad470faa21a67a2d1d6674",
            "5a8cf28b3d7846b4821c77cdbaab2e02",
            "81f3a475824149908de8f47f211f30a5",
            "56e762597c5e4e3c86dedf6f14aa93c4",
            "91d4742142934b8a989d9d65857787cc",
            "b3ed38ca0b1343e0a673eff945a9e99e",
            "f3118be89a0c438e8877ff837d4fd870",
            "26a9c6295aa2417eb2f82214e7071104",
            "cb5bcb118e0d4bc2bffaa833098a8005",
            "dd9a9a2f2bef4706abbba4efda8cfee1",
            "960e58fa76694ba6a0e5e9b9e41262d5",
            "7a8ca335433047868827a43d9c09cfe0",
            "6548aa080e6849db87e0411b0f2c5e34",
            "917e0eacc7b647488e052e990cea686e"
          ]
        },
        "id": "YEMw7RMQuGMn",
        "outputId": "7d7919b3-6506-4d1a-98f8-6814b822a74d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tokenizing...\n",
            "\n",
            "     "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#0:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "bb0114e630524f02953e7c415ba2fc3a"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#1:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "88b1f3a9e23d4db9865d7a11cacebb50"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#2:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4b03fa2191a54d9f88d693eb2cd4dbe3"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#3:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "91ab0629fd7a47119a1629b18d687f21"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Removing unanswerable questions...\n",
            "\n",
            "     "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#0:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1c25a2b2ccf845368b22a678a67a7cf1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#1:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "1faa01f2b1f248f6b941331e7a9228cc"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#2:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "e160666b91b149ec960f7e0eca9135ab"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#3:   0%|          | 0/22 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "4e98f41b8f2a40f4bc1d65e842ca0b90"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "\n",
            "Removed 3327 rows. (3.8765380313199107% of the dataset.)\n",
            "Tokenizing...\n",
            "\n",
            "     "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#0:   0%|          | 0/6 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "5a0dadd77b5346909bda88dc8afe3ae1"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#1:   0%|          | 0/6 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "81bf3840fa6f48808e09e67870b1c39c"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#2:   0%|          | 0/6 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "9ae329ef5d7b49d3a46a195cf9148f1f"
            }
          },
          "metadata": {}
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            " "
          ]
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "#3:   0%|          | 0/6 [00:00<?, ?ba/s]"
            ],
            "application/vnd.jupyter.widget-view+json": {
              "version_major": 2,
              "version_minor": 0,
              "model_id": "56e762597c5e4e3c86dedf6f14aa93c4"
            }
          },
          "metadata": {}
        }
      ],
      "source": [
        "input_tokenizer = AutoTokenizer.from_pretrained('distilroberta-base')\n",
        "tk_train = prepare_dataset(ds_train, input_tokenizer, input_len, out_len, out_tokenizer, delete_unans=True)\n",
        "tk_val = prepare_dataset(ds_val, input_tokenizer, input_len, out_len, out_tokenizer)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 24,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Zce-3anBwB-b",
        "outputId": "79af4e76-89c7-4575-f8bf-74798ebe58aa"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[0, 1779, 21, 5, 468, 415, 8207, 1357, 116, 2, 2, 133, 11484, 40201, 12589, 5672, 49038, 55, 10266, 373, 5, 11484, 5672, 50, 1622, 5, 468, 415, 6, 16, 5, 5560, 9, 5, 8427, 4250, 6, 2034, 11, 11484, 412, 4, 8575, 2368, 2885, 11, 501, 2545, 6, 1712, 24, 16, 203, 2530, 6, 24, 16, 65, 9, 5, 7763, 18146, 11, 5, 232, 8, 6308, 65, 9, 5, 144, 1233, 12245, 9, 4566, 14301, 4, 85, 34, 3337, 6, 151, 20993, 6355, 31, 1328, 750, 6, 25, 157, 25, 112, 4, 134, 153, 11118, 2799, 6, 61, 680, 103, 290, 6, 1497, 5853, 879, 873, 5571, 4, 1437, 50118, 50118, 133, 11484, 5672, 16, 10, 557, 5560, 13, 750, 6, 488, 6, 10561, 6, 2866, 8, 37007, 4, 20, 11484, 5672, 16, 490, 7, 1268, 54, 64, 3780, 49, 18952, 8, 557, 782, 4, 23769, 1975, 1517, 918, 13, 940, 892, 9, 6052, 31, 2799, 1027, 227, 504, 2663, 8, 4525, 64, 28, 5372, 11, 621, 50, 30, 7107, 4, 1437, 50118, 50118, 1121, 494, 777, 6, 5, 11484, 5672, 880, 41, 2557, 237, 12, 180, 695, 9, 16808, 3009, 63, 2783, 9, 42486, 6, 7, 28, 156, 577, 804, 4, 1437, 50118, 50118, 133, 11484, 8350, 26844, 58, 8254, 31, 5, 5560, 23, 5, 1786, 9, 5, 601, 212, 3220, 131, 51, 5585, 277, 3982, 6, 151, 1964, 4, 1437, 50118, 50118, 18077, 43018, 33, 10341, 6408, 5, 750, 9, 5, 5560, 88, 292, 5788, 6, 5048, 12, 42609, 260, 6, 6811, 260, 6, 5080, 4932, 261, 6, 5048, 12, 846, 47474, 8, 11484, 4, 1437, 50118, 50118, 133, 5048, 12, 42609, 260, 675, 6, 14096, 5, 2557, 360, 9, 5, 5560, 6, 7000, 31, 5, 13342, 360, 9, 5, 2197, 4, 4041, 10, 6095, 9, 7267, 6008, 31, 42, 675, 6, 600, 103, 32, 182, 1233, 4, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]\n",
            "[2, 16, 14, 11200, 2095, 11, 17425, 3, 0, 0]\n"
          ]
        }
      ],
      "source": [
        "print(tk_train[0]['input_ids'])\n",
        "print(tk_train[0]['labels'])"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "It's now time to create the trainer"
      ],
      "metadata": {
        "id": "r40_V6CrcXli"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 25,
      "metadata": {
        "id": "SShK-0ANwcLo"
      },
      "outputs": [],
      "source": [
        "trainer = Seq2SeqBertTrainer(encoder, decoder, out_len, out_tokenizer)"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "We will now fine tune the model for three epochs, this process takes awhile."
      ],
      "metadata": {
        "id": "akw7oSW4cbZ1"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 40,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 605
        },
        "id": "N99ojgpFxDgj",
        "outputId": "fe6c4195-bb81-4a81-c6ae-3cf74ff95a0d"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "10312it [1:30:54,  1.89it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished epoch 0, loss 2.3811252117156982\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "10312it [1:28:53,  1.93it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished epoch 1, loss 2.100656509399414\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "10312it [1:28:41,  1.94it/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Finished epoch 2, loss 1.930786371231079\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[<matplotlib.lines.Line2D at 0x7f7573c02040>]"
            ]
          },
          "metadata": {},
          "execution_count": 40
        },
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAeMAAAHTCAYAAAAUDXCVAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nO3dd3zV1eHG8efcTEJCWCFhhCUjsiEBmdZRByDiRAEBrYCICK7WVutPW60dWpWhIqAVENyoiKB1UA0ilCTsEbYoEPYIhISM8/uDa5siI4Ek547P+/XKy5vv/SZ5jt9cH8+9J+caa60AAIA7HtcBAAAIdpQxAACOUcYAADhGGQMA4BhlDACAY5QxAACOnbWMjTGJxpj5xpg1xpjVxpgxZzi3ozGmwBhzU9nGBAAgcIWW4JwCSQ9aazOMMTGS0o0xn1tr1xQ/yRgTIumvkv5Zkh9cs2ZN27Bhw9LmBQDAb6Wnp++11sadfPysZWyt3Slpp/d2tjFmraS6ktacdOq9kt6X1LEkgRo2bKi0tLSSnAoAQEAwxnx/quOles3YGNNQUntJi086XlfS9ZJePrd4AAAErxKXsTEmWidmvvdZaw+fdPcLkh621had5XsMN8akGWPS9uzZU/q0AAAEIFOSvamNMWGS5kj6zFr73Cnu3yLJeD+tKSlH0nBr7Yen+54pKSmWp6kBAMHEGJNurU05+fhZXzM2xhhJr0pae6oiliRrbaNi578uac6ZihgAAPxXSVZTd5M0SNJKY8wy77FHJNWXJGvtxHLKBgBAUCjJauoF+u9T0Gdlrb39fAIBABBs2IELAADHKGMAAByjjAEAcIwyBgDAMcoYAADHKGMAAByjjAEAcIwyBgDAMcoYAADHKGMAABwLmDLeuveo6wgAAJyTgCjjL9bs0mV//5c+XZXlOgoAAKUWEGXcvWlNta5XVQ+8s0xrdx52HQcAgFIJiDKODAvR5EHJiokM1dCpadp7JM91JAAASiwgyliSalWJ1OTBKdp7JE8j38jQ8YIi15EAACiRgCljSWpTr6r+dlMb/Xvrfv3fR6tkrXUdCQCAswp1HaCs9W1XV+t3ZevF+ZvUPCFGd3Rr5DoSAABnFFAz4588eEVzXdEiXk/OWaPUDXtcxwEA4IwCsow9HqPnb2mnprVidM+MDG3ec8R1JAAATisgy1iSoiNCNWVIikJDPBo6LU2HjuW7jgQAwCkFbBlLUmL1KL00sIO27cvR6DeXqrCIBV0AAN8T0GUsSZ0b19Af+7bS1+v36M9z17qOAwDAzwTcaupTGXBRfWVmHdaUBVvUPCFGN6ckuo4EAMB/BPzM+CePXdNC3ZrU0KMfrFL69/tdxwEA4D+CpoxDQzx6cUAH1a4aqbump2v7wWOuIwEAICmIyliSqkaFa8rgFOXmF2nY1DTlHC9wHQkAgOAqY0lqGh+j8f3ba23WYT307nK2zAQAOBd0ZSxJlybV0m+vTtLclVka9+VG13EAAEEuKFZTn8rwixsrMytbz3+xXs3io9WzdW3XkQAAQSooZ8aSZIzR0ze0Vvv6VfXAO8u1esch15EAAEEqaMtYkiLDQvTKbcmKrRSm4dPStfdInutIAIAgFNRlLEm1qkRq8uAU7TuapxHT05VXUOg6EgAgyAR9GUtS63qxeuamtkr7/oAe+3AVK6wBABUqaBdwnaxP2zpavytb47/aqOYJVXRn90auIwEAggQz42Lu/2UzXdUyXn/6ZI2+Xr/HdRwAQJCgjIvxeIye69dOzeJjNGpmhjbtOeI6EgAgCFDGJ6kcEarJg1MUFuLRsKlpOpST7zoSACDAUcankFg9ShNvS9YPB3I06s0MFRQWuY4EAAhglPFpdGpUXU/2baXUDXv19Nx1ruMAAAIYq6nP4NZO9bUuK1uvfbtFSQkx6tcx0XUkAEAAYmZ8Fr/vfaF6NK2pRz9cqSVb97uOAwAIQJTxWYSGeDShfwfVqxalEdPT9eOBHNeRAAABhjIugdioME0enKLjhUUaNi1dR/MKXEcCAAQQyriEmtSK1vj+7ZWZdVgPvrNcRUVsmQkAKBuUcSlc0ryWHul1oT5dnaUXvtzgOg4AIECwmrqU7uzeSOuysjXuyw1qHh+j3m1qu44EAPBzzIxLyRijP13fSskNqunBd5dp1fZDriMBAPwcZXwOIkJDNPG2ZFWPCtewaWnanZ3rOhIAwI9RxucoLiZCkwan6GBOvkZMT1deQaHrSAAAP0UZn4dWdWP1935tlbHtoB6ZtUrWssIaAFB6lPF56tW6tsZc3lTvZ/yoKalbXMcBAPghyrgMjLm8qXq2StCf563V/MzdruMAAPwMZVwGPB6jv/drq6SEKho9c6k27s52HQkA4Eco4zISFR6qyUNSFBHm0dCpaTqYc9x1JACAn6CMy1DdqpU08bZkbT94TKNmLlVBYZHrSAAAP0AZl7GUhtX1p+tba8HGvXrqk7Wu4wAA/ADbYZaDfimJyszK1qsLtqh5Qoz6d6rvOhIAwIcxMy4nv+uZpIubxemxD1dp8eZ9ruMAAHwYZVxOQkM8Gt+/vepXj9LdMzL0w/4c15EAAD6KMi5HsZXCNHlIivILizRsWpqO5hW4jgQA8EGUcTm7IC5aLw7ooPW7snX/28tUVMSWmQCA/0UZV4CLm8Xp0d4t9M81u/T8F+tdxwEA+BhWU1eQX3VrqMyswxr/1UY1i49Rn7Z1XEcCAPgIZsYVxBijJ69rpZQG1fTQu8u18sdDriMBAHwEZVyBIkJDNHFQsmpGR2jYtDTtPpzrOhIAwAdQxhWsZnSEJg1O1qFj+Ro+PV25+YWuIwEAHKOMHWhZJ1bP39JWy344qEdmrZS1rLAGgGBGGTtydavauv+XzTRr6XZN+maz6zgAAIcoY4dGX95EvVvX1l8+Xaev1u1yHQcA4Ahl7JAxRs/e3FYtalfR6DeXacOubNeRAAAOUMaOVQoP0eTBKYoMC9HQaWk6cPS460gAgApGGfuAOlUr6ZVBydp5MFf3zMxQfmGR60gAgApEGfuI5AbV9PQNrbVw0z49OWeN6zgAgArEdpg+5KbkesrMOqzJqVvUPCFGAy9q4DoSAKACMDP2Mb/teaEuaR6nxz9are827XMdBwBQAShjHxPiMRrXv70a1IjSyBnp2rYvx3UkAEA5o4x9UJXIME0Z0lFFVho2LU1H8gpcRwIAlCPK2Ec1qllZLw7ooI17jui+t5apqIgtMwEgUFHGPqx705p6rPeF+mLtLj37z0zXcQAA5eSsZWyMSTTGzDfGrDHGrDbGjDnFOX2NMSuMMcuMMWnGmO7lEzf4DOnaUP07Jeqlf23SR8u2u44DACgHJfnTpgJJD1prM4wxMZLSjTGfW2uL/zHsl5JmW2utMaaNpHckJZVD3qBjjNEfrm2lTXuO6jfvrVDDGpXVNrGq61gAgDJ01pmxtXantTbDeztb0lpJdU8654j97/sAVpbEC5xlKDzUo5cHdlBcTISGTUvTrsO5riMBAMpQqV4zNsY0lNRe0uJT3He9MWadpE8k/aoswuG/akRHaPLgFB3JK9DwaWnKzS90HQkAUEZKXMbGmGhJ70u6z1p7+OT7rbUfWGuTJF0n6cnTfI/h3teU0/bs2XOumYPWhbWr6Plb2mn5j4f08Psr9N8nIwAA/qxEZWyMCdOJIp5hrZ11pnOttd9IamyMqXmK+yZZa1OstSlxcXHnFDjYXdUyQQ9d2UwfLduhl7/e5DoOAKAMlGQ1tZH0qqS11trnTnNOE+95MsZ0kBQhib0cy8k9lzZRn7Z19MxnmfpizS7XcQAA56kkM+NukgZJusz7p0vLjDG9jDEjjDEjvOfcKGmVMWaZpBcl3WJ5DrXcGGP0txvbqFWdWI15a6kys7JdRwIAnAfjqjNTUlJsWlqak58dKLIO5arPhAWKDPPoo3u6q3rlcNeRAABnYIxJt9amnHycHbj8WEJspCYNStauw3kaOSNd+YVFriMBAM4BZezn2tevpr/e2FqLNu/XE7NXu44DADgHJdmBCz7u+vb1tC4rW698vVlJCTEa1KWh60gAgFJgZhwgfnNVki5LqqUnPl6jhRv3uo4DACgFyjhAhHiMxt7aTo1rVtbImRn6ft9R15EAACVEGQeQmMgwTRmSImuloVPTlJ2b7zoSAKAEKOMA06BGZb08sIM27z2q+95apsIi/twbAHwdZRyAujapqcf7tNCX63brmc8yXccBAJwFq6kD1KDODbQuK1sTv96k5gnRur59PdeRAACnwcw4QBlj9IdrW+qiRtX18PsrtXTbAdeRAACnQRkHsLAQj16+LVnxVSJ01/R0ZR3KdR0JAHAKlHGAq145XFMGd9TRvAINn56m3PxC15EAACehjINA84QYvXBre63cfki/fm+FeEMtAPAtlHGQuKJFvB66srk+Xr5DL/1rk+s4AIBiWE0dREZecoHW78rWM59lqmmtaF3ZMsF1JACAmBkHFWOM/npjG7WtF6v73l6mdVmHXUcCAIgyDjqRYSF6ZVCKoiNCNXRqmvYdyXMdCQCCHmUchBJiIzVpcIp2Z+fp7hkZOl5Q5DoSAAQ1yjhItUusqmduaqN/b9mvx2evZoU1ADjEAq4g1rddXa3LytbL/9qkpIQYDena0HUkAAhKzIyD3K+vbK5fXlhLf5yzRt9u3Os6DgAEJco4yHk8Ri/c2l4XxFXWyBkZ2rL3qOtIABB0KGMoOiJUUwZ3lMdIQ6cu0eHcfNeRACCoUMaQJNWvEaWXBibr+305Gv3mUhUWsaALACoKZYz/6HJBDT1xbUv9K3OP/vrpOtdxACBosJoa/+O2zg2UmZWtSd9sVrP4GN2UXM91JAAIeMyM8TP/16eFujSuoUdmrVT69wdcxwGAgEcZ42fCQjx6aWAH1a4aqbump2vHwWOuIwFAQKOMcUrVKodryuAU5eYXavj0NB07Xug6EgAELMoYp9U0Pkbj+rfT6h2H9dB7y9kyEwDKCWWMM7osKV4PX52kT1bs1PivNrqOAwABidXUOKu7Lm6s9VnZeu7z9WoWH62rW9V2HQkAAgozY5yVMUZP39Ba7RKr6v63l2vNjsOuIwFAQKGMUSKRYSGaNChZsZXCNGxamvYeyXMdCQACBmWMEqtVJVKTBidr75E83f1Guo4XFLmOBAABgTJGqbSpV1XP3txWS7Ye0GMfrmKFNQCUARZwodT6tK2jzKxsTZi/Uc0TYvSr7o1cRwIAv8bMGOfkgSua6coW8XrqkzX6Zv0e13EAwK9RxjgnHo/R87e0U7P4GI2amaHNe464jgQAfosyxjmrHBGqyYNTFBri0dBpaTp0LN91JADwS5Qxzkti9Si9NLCDtu3L0eg3l6qwiAVdAFBalDHOW+fGNfTHvq309fo9+vPcta7jAIDfYTU1ysSAi+orM+uwpizYouYJMbo5JdF1JADwG8yMUWYeu6aFujWpoUc/WKX07/e7jgMAfoMyRpkJDfHoxQEdVKdqpO6anq7tB4+5jgQAfoEyRpmqGhWuKUNSlJdfpGFT05RzvMB1JADweZQxylyTWjEa17+91mYd1kPvLlcRK6wB4IwoY5SLS5Nq6Xc9kzR3ZZbGfbXBdRwA8Gmspka5GdajsdZlZeuFLzaoeXyMerau7ToSAPgkZsYoN8YYPX19a7WvX1UPvLNcq3ccch0JAHwSZYxyFRkWolcGJatqVJiGTU3Tnuw815EAwOdQxih3tWIiNXlwivbnHNeIN9KVV1DoOhIA+BTKGBWiVd1YPXtzW6V/f0C//2CVrGWFNQD8hAVcqDDXtKmj9VnZGvfVRjVPiNHQHo1dRwIAn8DMGBXqvl8201Ut4/X03LX6ev0e13EAwCdQxqhQHo/Rc/3aqVl8jEbNzNCmPUdcRwIA5yhjVLjKEaGaMiRF4SEeDZ2apkM5+a4jAYBTlDGcqFctShMHJevHAzka9WaGCgqLXEcCAGcoYzjTsWF1PXVdK6Vu2Ks/zV3rOg4AOMNqajh1S8f6WpeVrX98u1VJCTG6pWN915EAoMIxM4Zzj/a6UD2a1tTvP1ylJVv3u44DABWOMoZzoSEeTejfQYnVojRierp+PJDjOhIAVCjKGD4hNipMk4ek6HhhkYZOTdPRvALXkQCgwlDG8BkXxEVrwoAOWr8rWw+8s0xFRWyZCSA4UMbwKb9oFqdHel2oz1bv0gtfrHcdBwAqBKup4XPu7N5Imd49rJslxOiaNnVcRwKAcsXMGD7HGKOnrm+l5AbV9NC7y7Vq+yHXkQCgXFHG8EkRoSGaeFuyqkeFa9i0NO3OznUdCQDKDWUMnxUXE6HJQ1J0MCdfd01PV25+oetIAFAuKGP4tJZ1YvVcv7Zauu2gHvlgpaxlhTWAwEMZw+f1bF1b9/2yqWZlbNeU1C2u4wBAmaOM4RdGX9ZUvVon6M/z1mp+5m7XcQCgTFHG8Asej9GzN7dVUkIVjZ65VBt3Z7uOBABlhjKG34gKD9XkISmKCPNo6NQ0Hcw57joSAJQJyhh+pW7VSnplULK2HzymUTOXqqCwyHUkADhvlDH8TnKD6vrT9a21YONePfXJWtdxAOC8sR0m/FK/lERlZmXr1QVb1DwhRv071XcdCQDOGTNj+K3f9UzSxc3i9NiHq7R48z7XcQDgnFHG8FuhIR6N799e9WtE6e4ZGfphf47rSABwTs5axsaYRGPMfGPMGmPMamPMmFOcM9AYs8IYs9IYs9AY07Z84gL/K7ZSmKYMTlFBYZGGTUvTkbwC15EAoNRKMjMukPSgtbaFpM6S7jHGtDjpnC2SfmGtbS3pSUmTyjYmcHqN46I1YUAHrd+VrfvfXqaiIrbMBOBfzlrG1tqd1toM7+1sSWsl1T3pnIXW2gPeTxdJqlfWQYEzubhZnH7fu4U+X7NLz32+3nUcACiVUq2mNsY0lNRe0uIznHanpHnnHgk4N3d0a6jMrGxNmL9RzRJidG3bOq4jAUCJlLiMjTHRkt6XdJ+19vBpzrlUJ8q4+2nuHy5puCTVr8+foqBsGWP05HWttHnvEf363eVqWCNKbepVdR0LAM6qRKupjTFhOlHEM6y1s05zThtJUyT1tdae8u9MrLWTrLUp1tqUuLi4c80MnFZ4qEcv35asmtERGj4tXbsP57qOBABnVZLV1EbSq5LWWmufO8059SXNkjTIWssLdnCqZnSEJg9O0eHcfA2bnq7c/ELXkQDgjEoyM+4maZCky4wxy7wfvYwxI4wxI7zn/J+kGpJe8t6fVl6BgZJoUaeKnuvXTst/OKjfzVopa1lhDcB3nfU1Y2vtAknmLOcMlTS0rEIBZeHqVgl64Ipmeu7z9WqeEKMRv7jAdSQAOCV24EJAu/eyJurdprb++uk6fbl2l+s4AHBKlDECmjFGz97UVi3rVNGYt5Zp/a5s15EA4GcoYwS8SuEhmjQoRZFhIRo6NU0Hjh53HQkA/gdljKBQp2olTRqcrKxDuRo5I0P5hUWuIwHAf1DGCBod6lfTn29ore8279MfP17jOg4A/EeptsME/N2NyfWUuStbk77ZrOYJMbqtcwPXkQCAmTGCz8NXJ+nS5nF6YvZqfbfplJvFAUCFoowRdEI8RmP7t1fDmpV194x0bduX4zoSgCBHGSMoVYkM05TBKbJWGjptibJz811HAhDEKGMErYY1K+ulgR20ac9R3f/2MhUWsWUmADcoYwS1bk1q6v+uaaEv1u7Ws//MdB0HQJBiNTWC3uAuDbQuK1sv/2uTmsfH6Lr2dV1HAhBkmBkj6Blj9IdrW6pTo+r6zfsrtOyHg64jAQgylDEgKTzUo4m3JatWTISGT0tT1qFc15EABBHKGPCqXjlcU4ak6GhegYZPT1NufqHrSACCBGUMFJOUUEXP39JOK7cf0sPvr5C1rLAGUP4oY+AkV7ZM0ENXNtdHy3bo5a83uY4DIAiwmho4hZGXXKDMrGw981mmmtWK0S9bxLuOBCCAMTMGTsEYo7/d1Eat68ZqzFtLlZmV7ToSgABGGQOnERkWokmDUlQ5IlRDpy3R/qPHXUcCEKAoY+AMEmIjNWlwinYdztPIGenKLyxyHQlAAKKMgbNol1hVf72xtRZt3q8nZq92HQdAAGIBF1AC17evp3VZ2Xrl681KSojRoC4NXUcCEECYGQMl9JurknRZUi098fEaLdy413UcAAGEMgZKKMRjNPbWdmpcs7JGzszQ9/uOuo4EIEBQxkApxESGacqQFEnSnVPTlJ2b7zgRgEBAGQOl1KBGZb00oIO27D2qMW8tU2ERW2YCOD+UMXAOujapqSf6tNBX63brb5+tcx0HgJ9jNTVwjgZ1afifFdbN42N0Q4d6riMB8FPMjIHz8MS1LdW5cXX9dtZKLd12wHUcAH6KMgbOQ1iIRy8NTFZ8lQgNn56unYeOuY4EwA9RxsB5ql45XK8O6aicvAINn5auY8cLXUcC4GcoY6AMNIuP0dhb22vVjkP6zfsrZC0rrAGUHGUMlJFftojXr69qro+X79CL8ze6jgPAj7CaGihDd//iAq3Pytaz/1yvpvExuqplgutIAPwAM2OgDBlj9Jcb26htvVjd//Yyrd152HUkAH6AMgbKWGRYiCYNTlFMZKiGTk3TviN5riMB8HGUMVAO4qtEatKgFO09kqe738jQ8YIi15EA+DDKGCgnbROr6m83tdG/t+7X47NXscIawGmxgAsoR33b1VVmVrZe+tcmNY+P0e3dGrmOBMAHMTMGytlDVzbXLy+M15OfrNWCDXtdxwHggyhjoJx5PEYv3NpOTeKiNXJGurbsPeo6EgAfQxkDFSA6IlRThqQoxGM0dOoSHc7Ndx0JgA+hjIEKklg9Si/flqzv9+Xo3plLVVjEgi4AJ1DGQAXq3LiG/tC3pb5ev0d/mbfWdRwAPoLV1EAFG3hRA2VmZWty6hY1T6iim5LruY4EwDFmxoADj13TQl0vqKFHZq1U+vf7XccB4BhlDDgQFuLRSwM7qHbVSN01PUM7Dh5zHQmAQ5Qx4EjVqHBNGZyi3PxCDZuWpmPHC11HAuAIZQw41DQ+RuP7t9eanYf10HvL2TITCFKUMeDYpUm19Nurk/TJip0a/9VG13EAOMBqasAHDL+4sTKzsvXc5+vVLD5aV7eq7ToSgArEzBjwAcYYPX1Da7WvX1X3v71ca3Ycdh0JQAWijAEfERkWolduS1ZspTANm5amvUfyXEcCUEEoY8CH1KoSqcmDU7TvaJ7ufiNdxwuKXEcCUAEoY8DHtK4Xq2duaqslWw/osQ9XscIaCAIs4AJ8UJ+2dZSZla0J8zeqeUKMftW9ketIAMoRM2PARz1wRTNd2SJeT32yRt+s3+M6DoByRBkDPsrjMXr+lnZqFh+jUTMztHnPEdeRAJQTyhjwYZUjQjV5cIpCQzwaOjVNh47lu44EoBxQxoCPS6wepZcHdtC2/Tm6982lKihkhTUQaChjwA9c1LiGnryulb5Zv0d/nrfOdRwAZYzV1ICf6N+pvjKzsvXqgi1qHh+jfh0TXUcCUEaYGQN+5Pe9L1T3JjX16IcrlbZ1v+s4AMoIZQz4kdAQjyYMaK+6VStpxBvp2n7wmOtIAMoAZQz4mapR4ZoypKPy8os0dGqaco4XuI4E4DxRxoAfalIrWuMGtFdm1mE9+M5yFRWxZSbgzyhjwE9d2ryWftfzQs1blaWxX25wHQfAeWA1NeDHhvZopMxd2Rr75QY1i49R7za1XUcCcA6YGQN+zBijP13fSh3qV9WD7y7Tqu2HXEcCcA4oY8DPRYSGaOKgZFWLCtfwaWnak53nOhKAUqKMgQBQKyZSkwenaH/Ocd01PU15BYWuIwEoBcoYCBCt6sbq7ze3U8a2g3r0g1WylhXWgL+gjIEA0rtNbY2+vKneS/9Rry7Y4joOgBKijIEAc9/lTdWzVYKenrtW8zN3u44DoAQoYyDAeDxGf+/XVs0Tqmj0zKXauPuI60gAzoIyBgJQVHioJg9OVnioR8OmpelQTr7rSADOgDIGAlS9alF6ZVCyfjyQo3tmZqigsMh1JACnQRkDASylYXX96brWWrBxr576ZK3rOABO46xlbIxJNMbMN8asMcasNsaMOcU5ScaY74wxecaYh8onKoBz0a9jon7VrZFeX7hVb/17m+s4AE6hJHtTF0h60FqbYYyJkZRujPncWrum2Dn7JY2WdF15hARwfh7plaSNe47osY9WqXFctDo1qu46EoBizjozttbutNZmeG9nS1orqe5J5+y21i6RxCoRwAeFhng0vn97JVaL0og30rVk637XkQAUU6rXjI0xDSW1l7T4XH6YMWa4MSbNGJO2Z8+ec/kWAM5RbKUwvXp7R8VEhuqWV77T+C83qJD3QQZ8QonL2BgTLel9SfdZaw+fyw+z1k6y1qZYa1Pi4uLO5VsAOA+NalbWnHu765o2dfT3z9dr0KuLtetwrutYQNArURkbY8J0oohnWGtnlW8kAOUpJjJMY29tp7/d2EYZ2w6o19hUduoCHCvJamoj6VVJa621z5V/JADlzRijfh0TNefe7oqLidAd/1iip+eu1fEC/hYZcMGc7Z1djDHdJaVKWinpp0fqI5LqS5K1dqIxJkFSmqQq3nOOSGpxpqezU1JSbFpa2nkPAMD5yc0v1FOfrNEbi7apbb1Yje/fQfVrRLmOBQQkY0y6tTblZ8ddvc0aZQz4lnkrd+o376+QrPT0Da3Vp20d15GAgHO6MmYHLgCSpJ6ta2vu6B5qGh+te99cqt/NWqFjxwtdxwKCAmUM4D8Sq0fp7bu6aOQlF+itJT/o2gkLlJmV7ToWEPAoYwD/IyzEo99cnaRpv+qkAznHde2EBZq5eJtcvaQFBAPKGMAp9Wgap7ljeqhTo+p65IOVGjVzqQ4dY5M9oDxQxgBOq1ZMpKbe0UkPX52kT1dnqfe4VC3ddsB1LCDgUMYAzsjjMbr7kgv0zl1dZK1088TvNPHrTSpiK02gzFDGAEokuUE1zR3TQ1e0iNdf5q3T7a8v0d4jea5jAQGBMgZQYrGVwvTSwA566rpWWrR5n3qOTdWCDXtdxwL8HmUMoFSMMbqtcwPNHtVNsZXCNOi1xXrms3UqKIXxYn8AABgDSURBVGQrTeBcUcYAzklSQhXNHtVN/ZIT9eL8Tbpl0iJtP3jMdSzAL1HGAM5ZVHio/npTG429tZ0ys7LV84Vv9OmqLNexAL9DGQM4b33b1dUno7urYc3KGvFGuh77cJVy89lKEygpyhhAmWhQo7LeG9FVQ7s30vRF3+u6F7/Vxt1HXMcC/AJlDKDMhId69PtrWui121O0OztPfcYv0LtpP7CVJnAWlDGAMndZUrzmju6htomx+vV7K3T/28t0JK/AdSzAZ1HGAMpFQmykZgztrAeuaKbZy3fomnGpWrX9kOtYgE+ijAGUmxCP0ejLm+rNYZ2VV1Ck61/6Vq8t2MLT1sBJKGMA5e6ixjU0d3QP/aJZLf1xzhoNnZqm/UePu44F+AzKGECFqFY5XJMHJ+vxPi2UumGveo1N1eLN+1zHAnwCZQygwhhjdEe3Rpo1sqsqhYeo/+RFeuGL9SrkHaAQ5ChjABWuVd1YfXxvd/VtV1cvfLFBAyYvUtahXNexAGcoYwBOREeE6vlb2unZm9tqxY+H1HPsN/pq3S7XsQAnKGMATt2UXE9zRndXQmwl/er1ND05Z43yCthKE8GFMgbg3AVx0fpgZFcN6dJAry7Yopte/k5b9x51HQuoMJQxAJ8QGRaiP/RtpVcGJWvb/hz1Hpeqj5Ztdx0LqBCUMQCfclXLBM0d00MX1q6iMW8t06/fXa6c42ylicBGGQPwOXWrVtJbwzvr3sua6L2MH9Vn/AKt3XnYdSyg3FDGAHxSaIhHD17ZXG/ceZEO5xao74vfavqi79lKEwGJMgbg07o1qal5Y3qoS+MaeuzDVbr7jQwdysl3HQsoU5QxAJ9XMzpC/7i9ox7plaQv1u5Sr3GpSv9+v+tYQJmhjAH4BY/HaPjFF+i9u7vK45H6vbJIL87fqCK20kQAoIwB+JV2iVX1yegeurpVgp75LFODX/u3dmezlSb8G2UMwO9UiQzThP7t9ZcbWivt+/3qNTZV36zf4zoWcM4oYwB+yRijWzvV1+xR3VW9crgGv/Zv/WXeOuUXFrmOBpQaZQzArzWLj9FH93RX/071NfHrTer3ynf6YX+O61hAqVDGAPxepfAQ/fmG1powoL027jqiXuNSNXflTtexgBKjjAEEjGva1NHcMT3UOC5aI2dk6NEPVio3n3eAgu+jjAEElMTqUXr3ri666+LGmrF4m/pO+FYbdmW7jgWcEWUMIOCEh3r0u14X6vU7OmrvkTz1mbBAby/Zxlaa8FmUMYCAdUnzWpo3poc61K+mh99fqdFvLVN2LltpwvdQxgACWq0qkZp+50X69VXNNXflTvUet0DLfzjoOhbwPyhjAAEvxGN0z6VN9PbwziooLNKNLy/U5G82s5UmfAZlDCBopDSsrrljeuiypFr609y1unPqEu07kuc6FkAZAwguVaPC9cqgZP2xb0t9u3Gfeo5N1cJNe13HQpCjjAEEHWOMBndpqA/u6aroyFANnLJYz/0zUwVspQlHKGMAQatlnVh9PKq7buxQT+O+2qgBkxdrx8FjrmMhCFHGAIJa5YhQPXtzWz1/S1ut2nFIvcal6vM1u1zHQpChjAFA0vXt62nOvd1Vt2olDZuWpidmr1ZeAVtpomJQxgDg1TguWrNGdtUd3Rrq9YVbdcNLC7V5zxHXsRAEKGMAKCYiNESP92mpyYNTtP3gMV0zfoFmZfzoOhYCHGUMAKdwRYt4zRvTQ63qxOqBd5brgXeW6WhegetYCFCUMQCcRu3YSpo57CKNubypPli6XX3GL9DqHYdcx0IAoowB4AxCQzy6/4pmmjm0s44eL9D1Ly7U1IVbeQcolCnKGABKoMsFNTR3dA91a1JDj89ereHT03Uw57jrWAgQlDEAlFCN6Ai9dntH/b73hfpX5m71GpuqJVv3u46FAEAZA0ApGGM0tEdjvX93V4WFenTLK99p/JcbVMg7QOE8UMYAcA7a1KuqOfd21zVt6ujvn6/XbVMWa9fhXNex4KcoYwA4RzGRYRp7azv97cY2WvrDAfUam6r5mbtdx4IfoowB4DwYY9SvY6Lm3NtdcTERuuMfS/T03LU6XsA7QKHkKGMAKANNasXow3u66bbO9TXpm826eeJCbduX4zoW/ARlDABlJDIsRE9d11ovD+ygzXuPqve4VH28fIfrWPADlDEAlLGerWtr7ugeahIfrXvfXKrfvr9Cx47zDlA4PcoYAMpBYvUovXNXF919yQV6O+0HXTthgTKzsl3Hgo+ijAGgnISFePTw1Uma9qtOOpBzXNdOWKCZi7exlSZ+hjIGgHLWo2mc5o7poU6NquuRD1Zq1MylOnQs33Us+BDKGAAqQK2YSE29o5MevjpJn67OUu9xqVq67YDrWPARlDEAVBCPx+juSy7QO3d1kbXSzRO/08SvN6mIrTSDHmUMABUsuUE1zR3TQ1e0iNdf5q3T7a8v0Z7sPNex4BBlDAAOxFYK00sDO+ip61pp0eZ96jUuVQs27HUdC45QxgDgiDFGt3VuoNmjuim2UpgGvbZYz3y2TgWFbKUZbChjAHAsKaGKZo/qpn7JiXpx/ibdMmmRfjzAVprBhDIGAB8QFR6qv97URmNvbafMrGz1GpuqT1ftdB0LFYQyBgAf0rddXX0yursa1qysEW9k6LEPVyk3n600Ax1lDAA+pkGNynpvRFcN7d5I0xd9r+te/FYbdx9xHQvliDIGAB8UHurR769podduT9Hu7Dz1Gb9A76b9wFaaAYoyBgAfdllSvOaO7qG2ibH69XsrdP/by3Qkr8B1LJQxyhgAfFxCbKRmDO2sB65optnLd+iacala+eMh17FQhihjAPADIR6j0Zc31ZvDOiuvoEg3vPytXl2whaetA8RZy9gYk2iMmW+MWWOMWW2MGXOKc4wxZpwxZqMxZoUxpkP5xAWA4HZR4xqaO7qHftGslp6cs0ZDp6Zp/9HjrmPhPJVkZlwg6UFrbQtJnSXdY4xpcdI5PSU19X4Ml/RymaYEAPxHtcrhmjw4WY/3aaHUDXvVa2yqFm3e5zoWzsNZy9hau9Nam+G9nS1praS6J53WV9I0e8IiSVWNMbXLPC0AQNKJrTTv6NZIs0Z2VWSYRwMmL9ILX6xXIe8A5ZdK9ZqxMaahpPaSFp90V11JPxT7/Ef9vLABAGWsVd1YzRndQ33b1dULX2zQgMmLlHUo13UslFKJy9gYEy3pfUn3WWsPn8sPM8YMN8akGWPS9uzZcy7fAgBwkuiIUD1/Szs9e3NbrfjxkHqO/UZfrt3lOhZKoURlbIwJ04kinmGtnXWKU7ZLSiz2eT3vsf9hrZ1krU2x1qbExcWdS14AwGnclFxPc0Z3V0JsJd05NU1//HiN8grYStMflGQ1tZH0qqS11trnTnPabEmDvauqO0s6ZK1lh3MAqGAXxEXrg5FdNaRLA7327Rbd9PJ32rr3qOtYOIuSzIy7SRok6TJjzDLvRy9jzAhjzAjvOXMlbZa0UdJkSSPLJy4A4Gwiw0L0h76t9MqgZG3bn6Pe41L10bKfPVkJH2Jc/cF4SkqKTUtLc/KzASBYbD94TGPeXKq07w/o5uR6+kPflooKD3UdK2gZY9KttSknH2cHLgAIYHWrVtJbwztr1KVN9F7Gj+ozfoHW7jynNbgoR5QxAAS40BCPHrqqud648yIdzi1Q3xe/1fTvtrKVpg+hjAEgSHRrUlPzxvRQ58Y19NhHq3X3Gxk6lJPvOhZEGQNAUKkZHaHXb++oR3ol6Yu1u9RrXKrSv9/vOlbQo4wBIMh4PEbDL75A793dVR6P1O+VRXpx/kYVsZWmM5QxAASpdolV9cnoHrq6ZYKe+SxTg1/7t3Zns5WmC5QxAASxKpFhmjCgvf58Q2st2bpfvcam6pv1bFdc0ShjAAhyxhj171RfH9/bXdUrh2vwa//WX+atU35hketoQYMyBgBIkprFx+ije7qrf6f6mvj1JvV75Tv9sD/HdaygQBkDAP6jUniI/nxDa00Y0F4bdx1Rr3GpmruStxoob5QxAOBnrmlTR3PH9FDjuGiNnJGhRz9Yqdx83gGqvFDGAIBTSqwepXfv6qK7Lm6sGYu3qe+Eb7VhV7brWAGJMgYAnFZ4qEe/63WhXr+jo/YeyVOfCQv09pJtbKVZxihjAMBZXdK8luaN6aEO9avp4fdXavRby5Sdy1aaZYUyBgCUSK0qkZp+50V66Mpmmrtyp3qPW6DlPxx0HSsgUMYAgBIL8RiNuqyp3h7eWQWFRbrx5YWa/M1mttI8T5QxAKDUUhpW19wxPXRZUi39ae5a3Tl1ifYdyXMdy29RxgCAc1I1KlyvDErWH/u21Lcb96nn2FQt3LTXdSy/RBkDAM6ZMUaDuzTUB/d0VXRkqAZOWazn/pmpArbSLBXKGABw3lrWidXHo7rrxg71NO6rjRowebF2HDzmOpbfoIwBAGWickSonr25rZ6/pa1W7TikXuNS9fmaXa5j+QXKGABQpq5vX09z7u2uulUradi0ND0xe7XyCthK80woYwBAmWscF61ZI7vqjm4N9frCrbrhpYXavOeI61g+izIGAJSLiNAQPd6npSYPTtH2g8d0zfgFmpXxo+tYPokyBgCUqytaxGvemB5qVSdWD7yzXA+8s0xH8wpcx/IplDEAoNzVjq2kmcMu0pjLm+qDpdvVZ/wCrd5xyHUsn0EZAwAqRGiIR/df0Uwzh3bW0eMFuv7FhZq6cCvvACXKGABQwbpcUENzR/dQtyY19Pjs1Ro+PV0Hc467juUUZQwAqHA1oiP02u0d9fveF+pfmbvVa2yqlmzd7zqWM5QxAMAJY4yG9mis9+/uqrBQj2555TuN/3KDCoPwHaAoYwCAU23qVdWce7urd5s6+vvn63XblMXadTjXdawKRRkDAJyLiQzTuFvb6W83ttHSHw6o19hUzc/c7TpWhaGMAQA+wRijfh0TNefe7oqLidAd/1iip+eu1fGCwH8HKMoYAOBTmtSK0Yf3dNNtnetr0jebdfPEhdq2L8d1rHJFGQMAfE5kWIieuq61Xh7YQZv3HlXvcan6ePkO17HKDWUMAPBZPVvX1tzRPdQkPlr3vrlUv31/hY4dD7x3gKKMAQA+LbF6lN65q4vuvuQCvbXkB107YYEys7JdxypTlDEAwOeFhXj08NVJmvarTjqQc1zXTligmYu3BcxWmpQxAMBvXNwsTnPH9FCnRtX1yAcrNWrmUh06lu861nmjjAEAfqVWTKSm3tFJv7m6uT5dnaXe41K1dNsB17HOC2UMAPA7Ho/RyEua6J27usha6eaJ32ni15tU5KdbaVLGAAC/ldygmuaO6aErWsTrL/PW6fbXl2hPdp7rWKVGGQMA/FpspTC9NLCDnrqulRZt3qde41K1YMNe17FKhTIGAPg9Y4xu69xAH93TTbGVwjTotcV65rN1Kij0j600KWMAQMC4sHYVzR7VTf2SE/Xi/E26ZdIi/XjA97fSpIwBAAElKjxUf72pjcbe2k6ZWdnqNTZVn67a6TrWGVHGAICA1LddXc25t7sa1KisEW9k6LEPVyk33ze30qSMAQABq2HNynr/7q4a2r2Rpi/6Xte9+K027j7iOtbPUMYAgIAWHurR769podduT9Guw7nqM36B3k37wae20qSMAQBB4bKkeM0bc7HaJsbq1++t0P1vL9ORvALXsSRRxgCAIJIQG6kZQzvrgSuaafbyHbpmXKpW/njIdSzKGAAQXEI8RqMvb6o3h3VWbn6Rbnj5W726YIvTp60pYwBAULqocQ3NG9NDv2gWpyfnrNHQqWnaf/S4kyyUMQAgaFWrHK7Jg1P0eJ8WSt2wV73GpmrR5n0VnoMyBgAENWOM7ujWSLNGdlVkmEcDJi/SC1+sV2EFvgMUZQwAgKRWdWM1Z3QP9W1XVy98sUEDJi+qsHeAoowBAPCKjgjV87e007M3t9WRvAJFhYdUyM8NrZCfAgCAH7kpuZ5uaF9XHo+pkJ/HzBgAgFOoqCKWKGMAAJyjjAEAcIwyBgDAMcoYAADHKGMAAByjjAEAcIwyBgDAMcoYAADHKGMAAByjjAEAcIwyBgDAMcoYAADHKGMAAByjjAEAcIwyBgDAMcoYAADHKGMAABwz1lo3P9iYPZK+L8NvWVPS3jL8fi4xFt8UKGMJlHFIjMVXBcpYymMcDay1cScfdFbGZc0Yk2atTXGdoywwFt8UKGMJlHFIjMVXBcpYKnIcPE0NAIBjlDEAAI4FUhlPch2gDDEW3xQoYwmUcUiMxVcFylgqbBwB85oxAAD+KpBmxgAA+CW/KGNjzNXGmExjzEZjzG9PcX+EMeZt7/2LjTENi933O+/xTGPMVRWZ+1RKMJYHjDFrjDErjDFfGmMaFLuv0BizzPsxu2KT/1wJxnK7MWZPscxDi903xBizwfsxpGKT/yzn2cbxfLExrDfGHCx2n89cE2PMa8aY3caYVae53xhjxnnHucIY06HYfT5zPbx5zjaWgd4xrDTGLDTGtC1231bv8WXGmLSKS31qJRjLJcaYQ8V+j/6v2H1n/N2saCUYy6+LjWOV9/FR3Xufz1wXY0yiMWa+97+1q40xY05xTsU+Xqy1Pv0hKUTSJkmNJYVLWi6pxUnnjJQ00Xv7Vklve2+38J4fIamR9/uE+PhYLpUU5b19909j8X5+xPX1KOVYbpc04RRfW13SZu8/q3lvV/PVcZx0/r2SXvPRa3KxpA6SVp3m/l6S5kkykjpLWuxr16MUY+n6U0ZJPX8ai/fzrZJqur4epRjLJZLmnOJ4qX43fWEsJ53bR9JXvnhdJNWW1MF7O0bS+lP896tCHy/+MDPuJGmjtXaztfa4pLck9T3pnL6SpnpvvyfpcmOM8R5/y1qbZ63dImmj9/u5ctaxWGvnW2tzvJ8uklSvgjOWVEmuy+lcJelza+1+a+0BSZ9Lurqccp5NacfRX9KbFZKslKy130jaf4ZT+kqaZk9YJKmqMaa2fOt6SDr7WKy1C71ZJd9+nJTkupzO+TzGykUpx+LLj5Wd1toM7+1sSWsl1T3ptAp9vPhDGdeV9EOxz3/Uz/+l/ecca22BpEOSapTwaytSafPcqRP/Z/aTSGNMmjFmkTHmuvIIWAolHcuN3qd43jPGJJbyaytCibN4XzJoJOmrYod96ZqczenG6kvX41yc/Dixkv5pjEk3xgx3lKm0uhhjlhtj5hljWnqP+e11McZE6URBvV/ssE9eF3PiZc32khafdFeFPl5Cz/cboHwYY26TlCLpF8UON7DWbjfGNJb0lTFmpbV2k5uEJfKxpDettXnGmLt04tmLyxxnOh+3SnrPWltY7Ji/XZOAYoy5VCfKuHuxw92916SWpM+NMeu8MzpflaETv0dHjDG9JH0oqanjTOerj6RvrbXFZ9E+d12MMdE68T8M91lrD7vM4g8z4+2SEot9Xs977JTnGGNCJcVK2lfCr61IJcpjjPmlpEclXWutzfvpuLV2u/efmyX9Syf+b86Vs47FWruvWP4pkpJL+rUVqDRZbtVJT7v52DU5m9ON1ZeuR4kZY9roxO9VX2vtvp+OF7smuyV9ILcvTZ2VtfawtfaI9/ZcSWHGmJry0+vidabHik9cF2NMmE4U8Qxr7axTnFKxjxfXL6Sf7UMnZu+bdeLpwZ8WMbQ86Zx79L8LuN7x3m6p/13AtVluF3CVZCztdWLRRtOTjleTFOG9XVPSBjlczFHCsdQudvt6SYu8t6tL2uIdUzXv7eq+Og7veUk6sQDF+Oo18eZoqNMvFOqt/12Q8m9fux6lGEt9nVgD0vWk45UlxRS7vVDS1T4+loSffq90oqC2ea9RiX43fWks3vtjdeJ15cq+el28/36nSXrhDOdU6OPF6UUtxb+4Xjqx2m2TpEe9x/6oEzNHSYqU9K73wflvSY2Lfe2j3q/LlNTTD8byhaRdkpZ5P2Z7j3eVtNL7gFwp6U4/GMufJa32Zp4vKanY1/7Ke702SrrDl8fh/fwJSX856et86proxExkp6R8nXgd605JIySN8N5vJL3oHedKSSm+eD1KOJYpkg4Ue5ykeY839l6P5d7fvUf9YCyjij1OFqnY/2Cc6nfTl8fiPed2nVg4W/zrfOq66MTLGlbSimK/Q71cPl7YgQsAAMf84TVjAAACGmUMAIBjlDEAAI5RxgAAOEYZAwDgGGUMAIBjlDEAAI5RxgAAOPb/cSuAN7UsJ+AAAAAASUVORK5CYII=\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ],
      "source": [
        "losses=[]\n",
        "for e in range(3):\n",
        "  epoch_loss=[]\n",
        "  for batch in tqdm(tk_train.iter(batch_size=bsize, drop_last_batch=True)):\n",
        "    actual_bsize = len(batch['input_ids'])\n",
        "    vals = {'encoder_input_ids': tf.reshape(tf.convert_to_tensor(batch['input_ids'], tf.int32), (bsize, input_len)),\n",
        "            'encoder_attention_mask': tf.reshape(tf.convert_to_tensor(batch['input_attention_mask'], tf.int32), (bsize, input_len)),\n",
        "            'decoder_target': tf.reshape(tf.convert_to_tensor(batch['labels'], tf.int32), (bsize, out_len))}\n",
        "    loss = trainer.batch_fit(vals)\n",
        "    epoch_loss.append(loss)\n",
        "  print(f'Finished epoch {e}, loss {np.mean(epoch_loss)}')\n",
        "  losses.append(np.mean(epoch_loss))\n",
        "plt.plot(losses)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 41,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Ruldpt0Paf4M",
        "outputId": "7966b19d-864d-4c2c-bcfe-a220c7558920"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Example 0\n",
            "Correct answer: ['<start> it was formally established in 1475 <end> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 1\n",
            "Correct answer: ['<start> research <end> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 2\n",
            "Correct answer: ['<start> history, and law <end> <UNK> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 3\n",
            "Correct answer: ['<start> philosophy, science and theology <end> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 4\n",
            "Correct answer: ['<start> a project <end> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 5\n",
            "Correct answer: ['<start> into periods <end> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 6\n",
            "Correct answer: ['<start> five <end> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 7\n",
            "Correct answer: ['<start> the vatican apostolic library <end> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 8\n",
            "Correct answer: ['<start> in vatican city <end> <UNK> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n",
            "Example 9\n",
            "Correct answer: ['<start> 1.1 million <end> <UNK> <UNK> <UNK> <UNK> <UNK> <UNK>']\n",
            "Predicted: ['no <end>']\n",
            "\n"
          ]
        }
      ],
      "source": [
        "for j in range(0, 10):\n",
        "  test = tk_train[j]['input_ids']\n",
        "  test = tf.reshape(tf.convert_to_tensor(test), (1, input_len))\n",
        "  ans = trainer.generate(test)\n",
        "  print(f'Example {j}\\nCorrect answer: {out_tokenizer.sequences_to_texts([tk_train[j][\"labels\"]])}\\nPredicted: {trainer.translate(ans)}\\n')"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "standard",
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.10.0"
    },
    "vscode": {
      "interpreter": {
        "hash": "afb734500600fd355917ca529030176ea0ca205570884b88f2f6f7d791fd3fbe"
      }
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "a68b3c6e4d2c4e8a8287a4e63291ff08": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_1ee5c0f410e54bda8d725d14db2a227a",
              "IPY_MODEL_b085728752cd4b1999dbab0eed63a3d7",
              "IPY_MODEL_538c138c74c24739b1bfeeae1966c8ed"
            ],
            "layout": "IPY_MODEL_18656f0b0870406c858e37adf5b5de8d"
          }
        },
        "1ee5c0f410e54bda8d725d14db2a227a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_361e3f9878b54b2fb37ab4416b0065cc",
            "placeholder": "​",
            "style": "IPY_MODEL_ba4be997d44349dd8a8d0144855aabc3",
            "value": "Downloading: 100%"
          }
        },
        "b085728752cd4b1999dbab0eed63a3d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c4811485725f4effbb0c589e4e1de54f",
            "max": 480,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e3f294fcee164f9681b3a025d9c12de7",
            "value": 480
          }
        },
        "538c138c74c24739b1bfeeae1966c8ed": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_20af1c404a284cf3827a86ad1b2efe88",
            "placeholder": "​",
            "style": "IPY_MODEL_06f70bbfd5794c2da03877d47cb9ad59",
            "value": " 480/480 [00:00&lt;00:00, 14.2kB/s]"
          }
        },
        "18656f0b0870406c858e37adf5b5de8d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "361e3f9878b54b2fb37ab4416b0065cc": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ba4be997d44349dd8a8d0144855aabc3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c4811485725f4effbb0c589e4e1de54f": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e3f294fcee164f9681b3a025d9c12de7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "20af1c404a284cf3827a86ad1b2efe88": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06f70bbfd5794c2da03877d47cb9ad59": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "6a6c4695bdf34597a417a9ca02da986b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_56e58362f95443109bc80500efe14c72",
              "IPY_MODEL_bc962701c4d1422784099213f1488f7f",
              "IPY_MODEL_145809341cc24978aa85d42dcd0f851c"
            ],
            "layout": "IPY_MODEL_b1ea85a8dab14fd6904131dfd83061f7"
          }
        },
        "56e58362f95443109bc80500efe14c72": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_35c0e742a3ab409c9738118738cae772",
            "placeholder": "​",
            "style": "IPY_MODEL_495a134a858141b6accb59d932a46be0",
            "value": "Downloading: 100%"
          }
        },
        "bc962701c4d1422784099213f1488f7f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f5623efe1f7846f4aa317b6494bbb0e7",
            "max": 898823,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_1f314aa9005347608422df3d99e58ff9",
            "value": 898823
          }
        },
        "145809341cc24978aa85d42dcd0f851c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c68fd39fd5d6439c9ebcc81702a68012",
            "placeholder": "​",
            "style": "IPY_MODEL_569e573164d445e8b11e7bb9034236e3",
            "value": " 878k/878k [00:01&lt;00:00, 1.05MB/s]"
          }
        },
        "b1ea85a8dab14fd6904131dfd83061f7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "35c0e742a3ab409c9738118738cae772": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "495a134a858141b6accb59d932a46be0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f5623efe1f7846f4aa317b6494bbb0e7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "1f314aa9005347608422df3d99e58ff9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "c68fd39fd5d6439c9ebcc81702a68012": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "569e573164d445e8b11e7bb9034236e3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d05a1653512b4de1aa331f967eca5999": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ac62dcc37eb74183b9dd96c1a6e685a1",
              "IPY_MODEL_54d6109e768b43dabea23b288aecd0d9",
              "IPY_MODEL_5fca857152f24325be0d9ad9b31edb65"
            ],
            "layout": "IPY_MODEL_537cbf9f13db4aad8a176729cddd35e1"
          }
        },
        "ac62dcc37eb74183b9dd96c1a6e685a1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb278481596f4a2b95f8dfb7f32a30a3",
            "placeholder": "​",
            "style": "IPY_MODEL_89955fa22feb4c6fa1b26e27c8767ad1",
            "value": "Downloading: 100%"
          }
        },
        "54d6109e768b43dabea23b288aecd0d9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e003729475824eb4a39d83c935b1de1b",
            "max": 456318,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_eeab9777039943b196895161d817f6ac",
            "value": 456318
          }
        },
        "5fca857152f24325be0d9ad9b31edb65": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f32d969478664ca9859a625bfd6d8a74",
            "placeholder": "​",
            "style": "IPY_MODEL_7eed1ff2bcd041519bd68b6f3cbc91f7",
            "value": " 446k/446k [00:01&lt;00:00, 515kB/s]"
          }
        },
        "537cbf9f13db4aad8a176729cddd35e1": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb278481596f4a2b95f8dfb7f32a30a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "89955fa22feb4c6fa1b26e27c8767ad1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e003729475824eb4a39d83c935b1de1b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "eeab9777039943b196895161d817f6ac": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "f32d969478664ca9859a625bfd6d8a74": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7eed1ff2bcd041519bd68b6f3cbc91f7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "55baab91f6c049aaa461e8254d50d7c8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_954567760e1749baa73f0be0868ef69b",
              "IPY_MODEL_d8ebaec53de3464d8f89eb8b3af92555",
              "IPY_MODEL_c9b36d60d41141ebb6f1e777e5194dc7"
            ],
            "layout": "IPY_MODEL_5a6c0f91e8de426495396f247bf7c053"
          }
        },
        "954567760e1749baa73f0be0868ef69b": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e2586e13500e438a9085bc53d27b68b8",
            "placeholder": "​",
            "style": "IPY_MODEL_c1e4b88424a44f2b815f0d7b48a26570",
            "value": "Downloading: 100%"
          }
        },
        "d8ebaec53de3464d8f89eb8b3af92555": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3b47033797024386800fd4e8af6668f8",
            "max": 1355863,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d5b409f3460c4f7e8a70273e5dd1fad9",
            "value": 1355863
          }
        },
        "c9b36d60d41141ebb6f1e777e5194dc7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_19e5356ca0f34e5f8f18329841c9f41a",
            "placeholder": "​",
            "style": "IPY_MODEL_5e2e21a173dc416cb488ea23ed1333c3",
            "value": " 1.29M/1.29M [00:01&lt;00:00, 1.05MB/s]"
          }
        },
        "5a6c0f91e8de426495396f247bf7c053": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e2586e13500e438a9085bc53d27b68b8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c1e4b88424a44f2b815f0d7b48a26570": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3b47033797024386800fd4e8af6668f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d5b409f3460c4f7e8a70273e5dd1fad9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "19e5356ca0f34e5f8f18329841c9f41a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5e2e21a173dc416cb488ea23ed1333c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "dc6190ef4e8f484cb0ec3f54d30b6793": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_cb1f485b70ca46b49b3f6dadee4adb3e",
              "IPY_MODEL_25ab2038b4064b5d87286688b9e10537",
              "IPY_MODEL_984a839dfef84665be4a738826b4549d"
            ],
            "layout": "IPY_MODEL_4b953f448bef4befbc98f4a465ad6c21"
          }
        },
        "cb1f485b70ca46b49b3f6dadee4adb3e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a51b51de8dd84c828aa78f100a7629d3",
            "placeholder": "​",
            "style": "IPY_MODEL_364fb9f621644346a126dfb4f80dd833",
            "value": "Downloading: 100%"
          }
        },
        "25ab2038b4064b5d87286688b9e10537": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_aca44a6d27ac49fe8f53578f117b1e09",
            "max": 331070498,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_e8daba08f0074044ba9cd722b579cf45",
            "value": 331070498
          }
        },
        "984a839dfef84665be4a738826b4549d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_635bc2ae55824a2881760f99b3315193",
            "placeholder": "​",
            "style": "IPY_MODEL_4c601f8ed9894ddb952828a67a2cb565",
            "value": " 316M/316M [00:04&lt;00:00, 73.3MB/s]"
          }
        },
        "4b953f448bef4befbc98f4a465ad6c21": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a51b51de8dd84c828aa78f100a7629d3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "364fb9f621644346a126dfb4f80dd833": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "aca44a6d27ac49fe8f53578f117b1e09": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e8daba08f0074044ba9cd722b579cf45": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "635bc2ae55824a2881760f99b3315193": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c601f8ed9894ddb952828a67a2cb565": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ed35025355c843629080d12fad8b215e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6b7d9855f305481fa64e7b96d3213221",
              "IPY_MODEL_338f45f7bff04e76a617d132c94f9128",
              "IPY_MODEL_ada1129519214046b9a2d7f7e8a7435d"
            ],
            "layout": "IPY_MODEL_fbb36868517548d89d0384e0c6564d3d"
          }
        },
        "6b7d9855f305481fa64e7b96d3213221": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_87edf2191c1247eab6136587cdcfb59d",
            "placeholder": "​",
            "style": "IPY_MODEL_001b597009914c36aee4114cf93127bf",
            "value": "100%"
          }
        },
        "338f45f7bff04e76a617d132c94f9128": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c7700c4a6d8b4398959fd83607e4b28e",
            "max": 85824,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b90309fc12264b669b9f7ff28fbf8716",
            "value": 85824
          }
        },
        "ada1129519214046b9a2d7f7e8a7435d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ecc7ace1d1614c559b6285a5c47269d7",
            "placeholder": "​",
            "style": "IPY_MODEL_6c296c5134674ca998607abc2773ff26",
            "value": " 85824/85824 [00:06&lt;00:00, 14653.56ex/s]"
          }
        },
        "fbb36868517548d89d0384e0c6564d3d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "87edf2191c1247eab6136587cdcfb59d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "001b597009914c36aee4114cf93127bf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c7700c4a6d8b4398959fd83607e4b28e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b90309fc12264b669b9f7ff28fbf8716": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "ecc7ace1d1614c559b6285a5c47269d7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6c296c5134674ca998607abc2773ff26": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a51158f0209b448ea592916964f6e7fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_380d5460dd2e4d8eaae19a9caa251954",
              "IPY_MODEL_593a92109ee64ea99ca26fa9042e7a9d",
              "IPY_MODEL_2892d2b28c3d40998b5127317038aadd"
            ],
            "layout": "IPY_MODEL_95309f66bf664e54b7c2a177ad6b61b6"
          }
        },
        "380d5460dd2e4d8eaae19a9caa251954": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_94c90de8d86f47168f5c76cc75b44bd7",
            "placeholder": "​",
            "style": "IPY_MODEL_e817dc63ea8f4d4dbb95317d03b5bbe3",
            "value": "100%"
          }
        },
        "593a92109ee64ea99ca26fa9042e7a9d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f36af37b73514f1184f115cd19f57c07",
            "max": 21452,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_61cc8aeb389d464b9f81dcf14d0edc95",
            "value": 21452
          }
        },
        "2892d2b28c3d40998b5127317038aadd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0bf16b30f9cf426ba668d8a466edbcb4",
            "placeholder": "​",
            "style": "IPY_MODEL_ac7b23622d6f40278d5d3f830ed8050d",
            "value": " 21452/21452 [00:01&lt;00:00, 14454.11ex/s]"
          }
        },
        "95309f66bf664e54b7c2a177ad6b61b6": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "94c90de8d86f47168f5c76cc75b44bd7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e817dc63ea8f4d4dbb95317d03b5bbe3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "f36af37b73514f1184f115cd19f57c07": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "61cc8aeb389d464b9f81dcf14d0edc95": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "0bf16b30f9cf426ba668d8a466edbcb4": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac7b23622d6f40278d5d3f830ed8050d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bb0114e630524f02953e7c415ba2fc3a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_6c1b789371084cc99831f8ee67decc1c",
              "IPY_MODEL_a40d37987e064ea2bd773c35666fcaab",
              "IPY_MODEL_1d081a07d0494faa8834b838d2c23566"
            ],
            "layout": "IPY_MODEL_8fdc7b9298464e13bfbc323912737a8a"
          }
        },
        "6c1b789371084cc99831f8ee67decc1c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_338589e812e848198db439fee3dfa790",
            "placeholder": "​",
            "style": "IPY_MODEL_db84f7a47d0a42a5ba602dc8fe602eb8",
            "value": "#0: 100%"
          }
        },
        "a40d37987e064ea2bd773c35666fcaab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c1c2a45c3cc416f85ce48d6cb0758c3",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_8b174ab0d1c743c98ccfa6ba422e72b6",
            "value": 22
          }
        },
        "1d081a07d0494faa8834b838d2c23566": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_47aa4f2d3f324f1d8ce7ff9a8c2cf411",
            "placeholder": "​",
            "style": "IPY_MODEL_23be94c3cd154d75b0b4e05077c27199",
            "value": " 22/22 [01:55&lt;00:00,  4.52s/ba]"
          }
        },
        "8fdc7b9298464e13bfbc323912737a8a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "338589e812e848198db439fee3dfa790": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "db84f7a47d0a42a5ba602dc8fe602eb8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3c1c2a45c3cc416f85ce48d6cb0758c3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8b174ab0d1c743c98ccfa6ba422e72b6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "47aa4f2d3f324f1d8ce7ff9a8c2cf411": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "23be94c3cd154d75b0b4e05077c27199": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "88b1f3a9e23d4db9865d7a11cacebb50": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7bda13cfd4e541e392124813bce25e4f",
              "IPY_MODEL_84372be77d5b4bceb832289eea82d600",
              "IPY_MODEL_b4b817b41d6c499394a2e2628d8ce18a"
            ],
            "layout": "IPY_MODEL_16a9ee1c222f480aa93b615cef08659c"
          }
        },
        "7bda13cfd4e541e392124813bce25e4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_13b681c3c4ed4c75befef92c225de3a7",
            "placeholder": "​",
            "style": "IPY_MODEL_64506eb279484e17872d1fe1c2b79c23",
            "value": "#1: 100%"
          }
        },
        "84372be77d5b4bceb832289eea82d600": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_913088feae2a4aa19cfd7cb87eae3c71",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_68d51c9637454ad98de1e836b167bbdf",
            "value": 22
          }
        },
        "b4b817b41d6c499394a2e2628d8ce18a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_974f022b4961402baede2ce2ed55917d",
            "placeholder": "​",
            "style": "IPY_MODEL_796abc7ded4b405ba9dda5df9efddf39",
            "value": " 22/22 [01:55&lt;00:00,  4.46s/ba]"
          }
        },
        "16a9ee1c222f480aa93b615cef08659c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "13b681c3c4ed4c75befef92c225de3a7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "64506eb279484e17872d1fe1c2b79c23": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "913088feae2a4aa19cfd7cb87eae3c71": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "68d51c9637454ad98de1e836b167bbdf": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "974f022b4961402baede2ce2ed55917d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "796abc7ded4b405ba9dda5df9efddf39": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4b03fa2191a54d9f88d693eb2cd4dbe3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_505001abdbf641dd9eb590a5c908adf3",
              "IPY_MODEL_da17f369ce54415e85f65021f7acc828",
              "IPY_MODEL_cc01844e033949c6bab33e728c6e0f9e"
            ],
            "layout": "IPY_MODEL_94050dfb9f0f41ea8df1b1f430dc8862"
          }
        },
        "505001abdbf641dd9eb590a5c908adf3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f14be8d1068e4ae8913583891f63bb3c",
            "placeholder": "​",
            "style": "IPY_MODEL_113820730fbb4bebabe2e5d20cef3d62",
            "value": "#2: 100%"
          }
        },
        "da17f369ce54415e85f65021f7acc828": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5e51abebe07b41729fbfcaf9301b2a3b",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_20ec9b04cb8a4193852cb0186748012f",
            "value": 22
          }
        },
        "cc01844e033949c6bab33e728c6e0f9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b752000f9fa94139bc11ed4c4870d1f8",
            "placeholder": "​",
            "style": "IPY_MODEL_b34a78cecc7844d1b4d4a254cb00894e",
            "value": " 22/22 [01:53&lt;00:00,  4.23s/ba]"
          }
        },
        "94050dfb9f0f41ea8df1b1f430dc8862": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f14be8d1068e4ae8913583891f63bb3c": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "113820730fbb4bebabe2e5d20cef3d62": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5e51abebe07b41729fbfcaf9301b2a3b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20ec9b04cb8a4193852cb0186748012f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b752000f9fa94139bc11ed4c4870d1f8": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b34a78cecc7844d1b4d4a254cb00894e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "91ab0629fd7a47119a1629b18d687f21": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ccbe8b01447444ccb226d47d2e801d51",
              "IPY_MODEL_41accc074902447b9155ce14847e1d00",
              "IPY_MODEL_1644cf3859be470599bc47e804e191e6"
            ],
            "layout": "IPY_MODEL_ed5df1bfbea849fe8d2ad0e7b370047e"
          }
        },
        "ccbe8b01447444ccb226d47d2e801d51": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_92c328913e6d43da9ef99a97f51f1f5d",
            "placeholder": "​",
            "style": "IPY_MODEL_5b76e0933f0144faa48438a25d0e9477",
            "value": "#3: 100%"
          }
        },
        "41accc074902447b9155ce14847e1d00": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d500cb3e1a614cb1b56007ba36a63af3",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4f9d14e67dfa47cda198d4fbd5119884",
            "value": 22
          }
        },
        "1644cf3859be470599bc47e804e191e6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b7ffc9558988439bbca7099f8d9da531",
            "placeholder": "​",
            "style": "IPY_MODEL_846146b0fe9d47d2bf795011f3438225",
            "value": " 22/22 [01:53&lt;00:00,  4.08s/ba]"
          }
        },
        "ed5df1bfbea849fe8d2ad0e7b370047e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "92c328913e6d43da9ef99a97f51f1f5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5b76e0933f0144faa48438a25d0e9477": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d500cb3e1a614cb1b56007ba36a63af3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4f9d14e67dfa47cda198d4fbd5119884": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b7ffc9558988439bbca7099f8d9da531": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "846146b0fe9d47d2bf795011f3438225": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1c25a2b2ccf845368b22a678a67a7cf1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7533494a57624d3397d3b9919793c8c6",
              "IPY_MODEL_2d748dc8d78a4b79bfc550eece411650",
              "IPY_MODEL_1560a247acff4b8c81c0004274a74920"
            ],
            "layout": "IPY_MODEL_ecf12d4c9e7f4a53aa14c7b9df2bc912"
          }
        },
        "7533494a57624d3397d3b9919793c8c6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_0c0325a3635240afb6bd07a1e92d6854",
            "placeholder": "​",
            "style": "IPY_MODEL_b2326eb1310149949aa6f23bf7bf41d7",
            "value": "#0: 100%"
          }
        },
        "2d748dc8d78a4b79bfc550eece411650": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4462da40e82d4d6b8dbb4e69e199498a",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_55d0fbe2f3bd42f7a22b8ee6b80a05c3",
            "value": 22
          }
        },
        "1560a247acff4b8c81c0004274a74920": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_52a32dbfcdb24921b135d7a397daaaf0",
            "placeholder": "​",
            "style": "IPY_MODEL_33721ab988bb4d6a84aef7316c9aeb4f",
            "value": " 22/22 [00:11&lt;00:00,  2.14ba/s]"
          }
        },
        "ecf12d4c9e7f4a53aa14c7b9df2bc912": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0c0325a3635240afb6bd07a1e92d6854": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2326eb1310149949aa6f23bf7bf41d7": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4462da40e82d4d6b8dbb4e69e199498a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "55d0fbe2f3bd42f7a22b8ee6b80a05c3": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "52a32dbfcdb24921b135d7a397daaaf0": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "33721ab988bb4d6a84aef7316c9aeb4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1faa01f2b1f248f6b941331e7a9228cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b2104de8d3a7410d984d9f72602e52f6",
              "IPY_MODEL_56d6feed048d43b1b350ff7a69d4255e",
              "IPY_MODEL_03d0a7a9428440eeb5d09f91ec0aa7a0"
            ],
            "layout": "IPY_MODEL_1a8a7e75a96b4d2bab933a9131e0d4fe"
          }
        },
        "b2104de8d3a7410d984d9f72602e52f6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_e9f79dff899a43f1bd1a8ec756439ebd",
            "placeholder": "​",
            "style": "IPY_MODEL_8de80b1a0e10469dae7ef39f39c929fd",
            "value": "#1: 100%"
          }
        },
        "56d6feed048d43b1b350ff7a69d4255e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1aa6c888c7324e4f9ed004ebe85183a3",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_908111a195ef4142a3828faec82676c9",
            "value": 22
          }
        },
        "03d0a7a9428440eeb5d09f91ec0aa7a0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1d486975f18b4aeab3ec11ad93619e2e",
            "placeholder": "​",
            "style": "IPY_MODEL_00aee941958741c1af04ccfe630a9da6",
            "value": " 22/22 [00:11&lt;00:00,  2.19ba/s]"
          }
        },
        "1a8a7e75a96b4d2bab933a9131e0d4fe": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e9f79dff899a43f1bd1a8ec756439ebd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8de80b1a0e10469dae7ef39f39c929fd": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1aa6c888c7324e4f9ed004ebe85183a3": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "908111a195ef4142a3828faec82676c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1d486975f18b4aeab3ec11ad93619e2e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "00aee941958741c1af04ccfe630a9da6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e160666b91b149ec960f7e0eca9135ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5e447b8656564f629866e7db4f5e148a",
              "IPY_MODEL_07af587a84e74543977bdf2a85cec029",
              "IPY_MODEL_821cf4378900472da5f2f3e270836668"
            ],
            "layout": "IPY_MODEL_188fbc1c331048afae97ffa32fda9e44"
          }
        },
        "5e447b8656564f629866e7db4f5e148a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_25e33db0d5b64408b358b855ff1f7dcd",
            "placeholder": "​",
            "style": "IPY_MODEL_4643f9b89ef7406183f817843c80c40d",
            "value": "#2: 100%"
          }
        },
        "07af587a84e74543977bdf2a85cec029": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2c450774bb5d4e70ad3a5b7f9fd6e707",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7a42ea74a2c74e0ea8b2b6bfc5e88003",
            "value": 22
          }
        },
        "821cf4378900472da5f2f3e270836668": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b370334a879f42c3829f6c5ae97a969d",
            "placeholder": "​",
            "style": "IPY_MODEL_e0f40ee5ea33461290ba269be1684da8",
            "value": " 22/22 [00:11&lt;00:00,  2.92ba/s]"
          }
        },
        "188fbc1c331048afae97ffa32fda9e44": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "25e33db0d5b64408b358b855ff1f7dcd": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4643f9b89ef7406183f817843c80c40d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2c450774bb5d4e70ad3a5b7f9fd6e707": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a42ea74a2c74e0ea8b2b6bfc5e88003": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b370334a879f42c3829f6c5ae97a969d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e0f40ee5ea33461290ba269be1684da8": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "4e98f41b8f2a40f4bc1d65e842ca0b90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3e15fb240e9847578096286fb9c1eb73",
              "IPY_MODEL_9012ab4cd5c448b5be013f83a0f52a98",
              "IPY_MODEL_21f550336293412780a8c8ba7cd90fd6"
            ],
            "layout": "IPY_MODEL_562e115ae6f14897b97829a6f1f43f4e"
          }
        },
        "3e15fb240e9847578096286fb9c1eb73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b2bd27bb7fea439e94471d464b5f1574",
            "placeholder": "​",
            "style": "IPY_MODEL_4c31fb54a1cf4af5bd03b8485cbdfc29",
            "value": "#3: 100%"
          }
        },
        "9012ab4cd5c448b5be013f83a0f52a98": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a07ecd50ffa4192a4a24fe499559d96",
            "max": 22,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d80342cea6344ae5b0cd910d8100a500",
            "value": 22
          }
        },
        "21f550336293412780a8c8ba7cd90fd6": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_99b2f203217b43929c9c9ad049424b11",
            "placeholder": "​",
            "style": "IPY_MODEL_54ccff485e7347c18e4e12522b2ded90",
            "value": " 22/22 [00:11&lt;00:00,  3.13ba/s]"
          }
        },
        "562e115ae6f14897b97829a6f1f43f4e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2bd27bb7fea439e94471d464b5f1574": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4c31fb54a1cf4af5bd03b8485cbdfc29": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a07ecd50ffa4192a4a24fe499559d96": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d80342cea6344ae5b0cd910d8100a500": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "99b2f203217b43929c9c9ad049424b11": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "54ccff485e7347c18e4e12522b2ded90": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5a0dadd77b5346909bda88dc8afe3ae1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_2c048fc882d54b638235417a1f122638",
              "IPY_MODEL_b458917039134768a057b9e4dd93735e",
              "IPY_MODEL_f946aa0f31c844bc8e78b788314e1e73"
            ],
            "layout": "IPY_MODEL_50a5fd4fe2274f9388dbf0dfb5f384d9"
          }
        },
        "2c048fc882d54b638235417a1f122638": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4786cdc6a7d644ba826a43b226c851ef",
            "placeholder": "​",
            "style": "IPY_MODEL_e65c1356bdac466ea7cd223494d4114e",
            "value": "#0: 100%"
          }
        },
        "b458917039134768a057b9e4dd93735e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_7347da795d394cae91b452dc0ca58027",
            "max": 6,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_6a0f660bcd99440dbf99002f64e790cc",
            "value": 6
          }
        },
        "f946aa0f31c844bc8e78b788314e1e73": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_37c8e14302dc4cf39189ff9937b95366",
            "placeholder": "​",
            "style": "IPY_MODEL_88cbe7373e4146c3b98e000edf22bfbb",
            "value": " 6/6 [00:28&lt;00:00,  4.24s/ba]"
          }
        },
        "50a5fd4fe2274f9388dbf0dfb5f384d9": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4786cdc6a7d644ba826a43b226c851ef": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "e65c1356bdac466ea7cd223494d4114e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "7347da795d394cae91b452dc0ca58027": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "6a0f660bcd99440dbf99002f64e790cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "37c8e14302dc4cf39189ff9937b95366": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "88cbe7373e4146c3b98e000edf22bfbb": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "81bf3840fa6f48808e09e67870b1c39c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_b950fe152e094ccd8eb098dbe48f6766",
              "IPY_MODEL_8c1ed1b009384f3998d3fa7071365d33",
              "IPY_MODEL_6dadc0e8ab174a919bdf994602747df9"
            ],
            "layout": "IPY_MODEL_4812fef0354a4cb6ae35d637418c57c7"
          }
        },
        "b950fe152e094ccd8eb098dbe48f6766": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_96966eb9e87949e2bf41d4253234f29a",
            "placeholder": "​",
            "style": "IPY_MODEL_3b354d90adac4050aa98393968d8a8ab",
            "value": "#1: 100%"
          }
        },
        "8c1ed1b009384f3998d3fa7071365d33": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bda1f554b98245938111c0ecdfd5508b",
            "max": 6,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_06a7cec6c0134983a8c9c273590f5920",
            "value": 6
          }
        },
        "6dadc0e8ab174a919bdf994602747df9": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1ac2f62152f34a169c3240199d96f24e",
            "placeholder": "​",
            "style": "IPY_MODEL_a84cfd4d64dc40a4860683175d109b03",
            "value": " 6/6 [00:27&lt;00:00,  4.07s/ba]"
          }
        },
        "4812fef0354a4cb6ae35d637418c57c7": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "96966eb9e87949e2bf41d4253234f29a": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3b354d90adac4050aa98393968d8a8ab": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "bda1f554b98245938111c0ecdfd5508b": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "06a7cec6c0134983a8c9c273590f5920": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "1ac2f62152f34a169c3240199d96f24e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a84cfd4d64dc40a4860683175d109b03": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9ae329ef5d7b49d3a46a195cf9148f1f": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_bba32c0c50684372962be5b3c4be3c2c",
              "IPY_MODEL_a6ca682e5efa4338b54537f631c4696a",
              "IPY_MODEL_fd247caa30c74b1daef472d59d29be36"
            ],
            "layout": "IPY_MODEL_41f41df8445d4bd9bab92efeb3833796"
          }
        },
        "bba32c0c50684372962be5b3c4be3c2c": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_3c2aaed9c90b48c1be2162983906988e",
            "placeholder": "​",
            "style": "IPY_MODEL_a461c15ab5784a7bb67073a21fe6ce0d",
            "value": "#2: 100%"
          }
        },
        "a6ca682e5efa4338b54537f631c4696a": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2b3c7313fbb54fb5b0fae56669767a5d",
            "max": 6,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_20feca53a7ad470faa21a67a2d1d6674",
            "value": 6
          }
        },
        "fd247caa30c74b1daef472d59d29be36": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5a8cf28b3d7846b4821c77cdbaab2e02",
            "placeholder": "​",
            "style": "IPY_MODEL_81f3a475824149908de8f47f211f30a5",
            "value": " 6/6 [00:27&lt;00:00,  3.95s/ba]"
          }
        },
        "41f41df8445d4bd9bab92efeb3833796": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3c2aaed9c90b48c1be2162983906988e": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a461c15ab5784a7bb67073a21fe6ce0d": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "2b3c7313fbb54fb5b0fae56669767a5d": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "20feca53a7ad470faa21a67a2d1d6674": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5a8cf28b3d7846b4821c77cdbaab2e02": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81f3a475824149908de8f47f211f30a5": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "56e762597c5e4e3c86dedf6f14aa93c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HBoxModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_91d4742142934b8a989d9d65857787cc",
              "IPY_MODEL_b3ed38ca0b1343e0a673eff945a9e99e",
              "IPY_MODEL_f3118be89a0c438e8877ff837d4fd870"
            ],
            "layout": "IPY_MODEL_26a9c6295aa2417eb2f82214e7071104"
          }
        },
        "91d4742142934b8a989d9d65857787cc": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cb5bcb118e0d4bc2bffaa833098a8005",
            "placeholder": "​",
            "style": "IPY_MODEL_dd9a9a2f2bef4706abbba4efda8cfee1",
            "value": "#3: 100%"
          }
        },
        "b3ed38ca0b1343e0a673eff945a9e99e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "FloatProgressModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_960e58fa76694ba6a0e5e9b9e41262d5",
            "max": 6,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7a8ca335433047868827a43d9c09cfe0",
            "value": 6
          }
        },
        "f3118be89a0c438e8877ff837d4fd870": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "HTMLModel",
          "model_module_version": "1.5.0",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_6548aa080e6849db87e0411b0f2c5e34",
            "placeholder": "​",
            "style": "IPY_MODEL_917e0eacc7b647488e052e990cea686e",
            "value": " 6/6 [00:26&lt;00:00,  3.80s/ba]"
          }
        },
        "26a9c6295aa2417eb2f82214e7071104": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cb5bcb118e0d4bc2bffaa833098a8005": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dd9a9a2f2bef4706abbba4efda8cfee1": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "960e58fa76694ba6a0e5e9b9e41262d5": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7a8ca335433047868827a43d9c09cfe0": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "ProgressStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "6548aa080e6849db87e0411b0f2c5e34": {
          "model_module": "@jupyter-widgets/base",
          "model_name": "LayoutModel",
          "model_module_version": "1.2.0",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "917e0eacc7b647488e052e990cea686e": {
          "model_module": "@jupyter-widgets/controls",
          "model_name": "DescriptionStyleModel",
          "model_module_version": "1.5.0",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}