EL.ipynb

{
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "138778c4",
      "metadata": {
        "id": "138778c4"
      },
      "outputs": [],
      "source": [
        "from tqdm import tqdm\n",
        "import pandas as pd\n",
        "from sklearn import metrics\n",
        "from scipy.spatial.distance import cdist"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "dTSILRD7hIHG",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dTSILRD7hIHG",
        "outputId": "9d7677f8-fc7d-403c-e8dd-6135bc581c01"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Collecting transformers\n",
            "  Downloading transformers-4.24.0-py3-none-any.whl (5.5 MB)\n",
            "\u001b[K     |████████████████████████████████| 5.5 MB 4.7 MB/s \n",
            "\u001b[?25hCollecting huggingface-hub<1.0,>=0.10.0\n",
            "  Downloading huggingface_hub-0.10.1-py3-none-any.whl (163 kB)\n",
            "\u001b[K     |████████████████████████████████| 163 kB 88.9 MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.7/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (1.21.6)\n",
            "Requirement already satisfied: importlib-metadata in /usr/local/lib/python3.7/dist-packages (from transformers) (4.13.0)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.7/dist-packages (from transformers) (6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.7/dist-packages (from transformers) (21.3)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.7/dist-packages (from transformers) (2022.6.2)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.7/dist-packages (from transformers) (3.8.0)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.7/dist-packages (from transformers) (4.64.1)\n",
            "Collecting tokenizers!=0.11.3,<0.14,>=0.11.1\n",
            "  Downloading tokenizers-0.13.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (7.6 MB)\n",
            "\u001b[K     |████████████████████████████████| 7.6 MB 89.1 MB/s \n",
            "\u001b[?25hRequirement already satisfied: typing-extensions>=3.7.4.3 in /usr/local/lib/python3.7/dist-packages (from huggingface-hub<1.0,>=0.10.0->transformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /usr/local/lib/python3.7/dist-packages (from packaging>=20.0->transformers) (3.0.9)\n",
            "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.7/dist-packages (from importlib-metadata->transformers) (3.10.0)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (2022.9.24)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests->transformers) (3.0.4)\n",
            "Installing collected packages: tokenizers, huggingface-hub, transformers\n",
            "Successfully installed huggingface-hub-0.10.1 tokenizers-0.13.1 transformers-4.24.0\n"
          ]
        }
      ],
      "source": [
        "!pip install transformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "70512d4e",
      "metadata": {
        "id": "70512d4e",
        "outputId": "ebc36a95-bb47-425a-9c2c-7456141778e6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "huggingface/tokenizers: The current process just got forked, after parallelism has already been used. Disabling parallelism to avoid deadlocks...\n",
            "To disable this warning, you can either:\n",
            "\t- Avoid using `tokenizers` before the fork if possible\n",
            "\t- Explicitly set the environment variable TOKENIZERS_PARALLELISM=(true | false)\n",
            "Collecting parallelformers\n",
            "  Downloading parallelformers-1.2.7.tar.gz (48 kB)\n",
            "\u001b[K     |████████████████████████████████| 48 kB 557 kB/s eta 0:00:011\n",
            "\u001b[?25hRequirement already satisfied: transformers>=4.2 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from parallelformers) (4.24.0)\n",
            "Requirement already satisfied: torch in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from parallelformers) (1.11.0)\n",
            "Collecting dacite\n",
            "  Downloading dacite-1.6.0-py3-none-any.whl (12 kB)\n",
            "Requirement already satisfied: filelock in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (3.6.0)\n",
            "Requirement already satisfied: packaging>=20.0 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (21.3)\n",
            "Requirement already satisfied: huggingface-hub<1.0,>=0.10.0 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (0.10.1)\n",
            "Requirement already satisfied: tqdm>=4.27 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (4.64.1)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (2022.7.25)\n",
            "Requirement already satisfied: requests in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (2.27.1)\n",
            "Requirement already satisfied: pyyaml>=5.1 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (6.0)\n",
            "Requirement already satisfied: numpy>=1.17 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (1.21.5)\n",
            "Requirement already satisfied: tokenizers!=0.11.3,<0.14,>=0.11.1 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from transformers>=4.2->parallelformers) (0.13.1)\n",
            "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from huggingface-hub<1.0,>=0.10.0->transformers>=4.2->parallelformers) (4.1.1)\n",
            "Requirement already satisfied: pyparsing!=3.0.5,>=2.0.2 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from packaging>=20.0->transformers>=4.2->parallelformers) (2.4.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (2.0.4)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (2021.10.8)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (1.26.9)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /home2/sashank.sridhar/miniconda3/envs/TripletLoss/lib/python3.9/site-packages (from requests->transformers>=4.2->parallelformers) (3.3)\n",
            "Building wheels for collected packages: parallelformers\n",
            "  Building wheel for parallelformers (setup.py) ... \u001b[?25ldone\n",
            "\u001b[?25h  Created wheel for parallelformers: filename=parallelformers-1.2.7-py3-none-any.whl size=117791 sha256=ce1c96f5d462c55210041d65abcc897059cc1349cfafe5afcc489f60b6fbb7c6\n",
            "  Stored in directory: /home2/sashank.sridhar/.cache/pip/wheels/4f/19/42/8d74380c84a1e93401ee163f3bd545853051a4d895fd95ca2e\n",
            "Successfully built parallelformers\n",
            "Installing collected packages: dacite, parallelformers\n",
            "Successfully installed dacite-1.6.0 parallelformers-1.2.7\n"
          ]
        }
      ],
      "source": [
        "!pip install parallelformers"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6b2ace9d",
      "metadata": {
        "id": "6b2ace9d"
      },
      "outputs": [],
      "source": [
        "!pip install faiss-gpu"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "5a0d2481",
      "metadata": {
        "id": "5a0d2481"
      },
      "source": [
        "Download snomed term-concept file from UMLS website"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ea498c9d",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ea498c9d",
        "outputId": "49723e0b-aee4-46f9-b162-e319f10475d6"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1569232\n"
          ]
        }
      ],
      "source": [
        "snomed_csv = pd.read_csv(\"data/sct2_Description_Snapshot-en_INT_20220831.txt\", delimiter=\"\\t\")\n",
        "print(len(snomed_csv))"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d811cd3c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "d811cd3c",
        "outputId": "2bce882d-ab28-4af6-b07f-ed73b1b5f4c8"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "Index(['id', 'effectiveTime', 'active', 'moduleId', 'conceptId',\n",
              "       'languageCode', 'typeId', 'term', 'caseSignificanceId'],\n",
              "      dtype='object')"
            ]
          },
          "execution_count": 3,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "snomed_csv.columns"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3327f0d9",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 530
        },
        "id": "3327f0d9",
        "outputId": "7bbd5033-5116-4b65-c66a-f2cb3365f859"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>effectiveTime</th>\n",
              "      <th>active</th>\n",
              "      <th>moduleId</th>\n",
              "      <th>conceptId</th>\n",
              "      <th>languageCode</th>\n",
              "      <th>typeId</th>\n",
              "      <th>term</th>\n",
              "      <th>caseSignificanceId</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>101013</td>\n",
              "      <td>20170731</td>\n",
              "      <td>1</td>\n",
              "      <td>900000000000207008</td>\n",
              "      <td>126813005</td>\n",
              "      <td>en</td>\n",
              "      <td>900000000000013009</td>\n",
              "      <td>Neoplasm of anterior aspect of epiglottis</td>\n",
              "      <td>900000000000448009</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>102018</td>\n",
              "      <td>20170731</td>\n",
              "      <td>1</td>\n",
              "      <td>900000000000207008</td>\n",
              "      <td>126814004</td>\n",
              "      <td>en</td>\n",
              "      <td>900000000000013009</td>\n",
              "      <td>Neoplasm of junctional region of epiglottis</td>\n",
              "      <td>900000000000448009</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>103011</td>\n",
              "      <td>20170731</td>\n",
              "      <td>1</td>\n",
              "      <td>900000000000207008</td>\n",
              "      <td>126815003</td>\n",
              "      <td>en</td>\n",
              "      <td>900000000000013009</td>\n",
              "      <td>Neoplasm of lateral wall of oropharynx</td>\n",
              "      <td>900000000000448009</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>104017</td>\n",
              "      <td>20170731</td>\n",
              "      <td>1</td>\n",
              "      <td>900000000000207008</td>\n",
              "      <td>126816002</td>\n",
              "      <td>en</td>\n",
              "      <td>900000000000013009</td>\n",
              "      <td>Neoplasm of posterior wall of oropharynx</td>\n",
              "      <td>900000000000448009</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>105016</td>\n",
              "      <td>20170731</td>\n",
              "      <td>1</td>\n",
              "      <td>900000000000207008</td>\n",
              "      <td>126817006</td>\n",
              "      <td>en</td>\n",
              "      <td>900000000000013009</td>\n",
              "      <td>Neoplasm of esophagus</td>\n",
              "      <td>900000000000448009</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "       id  effectiveTime  active            moduleId  conceptId languageCode  \\\n",
              "0  101013       20170731       1  900000000000207008  126813005           en   \n",
              "1  102018       20170731       1  900000000000207008  126814004           en   \n",
              "2  103011       20170731       1  900000000000207008  126815003           en   \n",
              "3  104017       20170731       1  900000000000207008  126816002           en   \n",
              "4  105016       20170731       1  900000000000207008  126817006           en   \n",
              "\n",
              "               typeId                                         term  \\\n",
              "0  900000000000013009    Neoplasm of anterior aspect of epiglottis   \n",
              "1  900000000000013009  Neoplasm of junctional region of epiglottis   \n",
              "2  900000000000013009       Neoplasm of lateral wall of oropharynx   \n",
              "3  900000000000013009     Neoplasm of posterior wall of oropharynx   \n",
              "4  900000000000013009                        Neoplasm of esophagus   \n",
              "\n",
              "   caseSignificanceId  \n",
              "0  900000000000448009  \n",
              "1  900000000000448009  \n",
              "2  900000000000448009  \n",
              "3  900000000000448009  \n",
              "4  900000000000448009  "
            ]
          },
          "execution_count": 4,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "snomed_csv.head()"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "eee36071",
      "metadata": {
        "id": "eee36071"
      },
      "source": [
        "Process snomed terms"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "fc74afa8",
      "metadata": {
        "id": "fc74afa8"
      },
      "outputs": [],
      "source": [
        "all_ids = snomed_csv['conceptId']\n",
        "all_names = []\n",
        "for i in snomed_csv['term']:\n",
        "    try:\n",
        "        all_names.append(i.lower())\n",
        "    except:\n",
        "        all_names.append('not applicable')\n",
        "#         print(i)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ecbc8292",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ecbc8292",
        "outputId": "eb0c0228-aa13-4a37-e4d5-d1d5cee77806"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "id                            1491117014\n",
              "effectiveTime                   20030131\n",
              "active                                 1\n",
              "moduleId              900000000000207008\n",
              "conceptId                      385432009\n",
              "languageCode                          en\n",
              "typeId                900000000000013009\n",
              "term                                 NaN\n",
              "caseSignificanceId    900000000000020002\n",
              "Name: 906846, dtype: object"
            ]
          },
          "execution_count": 6,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "snomed_csv.iloc[906846]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "6d11f0d6",
      "metadata": {
        "id": "6d11f0d6"
      },
      "outputs": [],
      "source": [
        "snomed_name_id = [(all_names[i], all_ids[i]) for i in range(len(all_ids))]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f61e031c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f61e031c",
        "outputId": "3a768eb2-906b-4974-d468-c53b63cd057e"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "1569232"
            ]
          },
          "execution_count": 8,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "len(all_ids)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8b2c1e53",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8b2c1e53",
        "outputId": "84851594-3060-4197-9886-78d9d1e443fa"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "['neoplasm of anterior aspect of epiglottis',\n",
              " 'neoplasm of junctional region of epiglottis',\n",
              " 'neoplasm of lateral wall of oropharynx',\n",
              " 'neoplasm of posterior wall of oropharynx',\n",
              " 'neoplasm of esophagus',\n",
              " 'neoplasm of cervical esophagus',\n",
              " 'neoplasm of thoracic esophagus',\n",
              " 'neoplasm of abdominal esophagus',\n",
              " 'neoplasm of middle third of esophagus',\n",
              " 'neoplasm of lower third of esophagus']"
            ]
          },
          "execution_count": 9,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "all_names[:10]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4de928c7",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4de928c7",
        "outputId": "1f491d1a-1d2a-451c-86db-12b1bc92c536"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "0    126813005\n",
              "1    126814004\n",
              "2    126815003\n",
              "3    126816002\n",
              "4    126817006\n",
              "5    126818001\n",
              "6    126819009\n",
              "7    126820003\n",
              "8    126822006\n",
              "9    126823001\n",
              "Name: conceptId, dtype: int64"
            ]
          },
          "execution_count": 10,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "all_ids[:10]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "0b808263",
      "metadata": {
        "id": "0b808263"
      },
      "source": [
        "# load sapbert"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "a7c7ac5b",
      "metadata": {
        "id": "a7c7ac5b"
      },
      "outputs": [],
      "source": [
        "import numpy as np\n",
        "import torch"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d2c96dea",
      "metadata": {
        "id": "d2c96dea"
      },
      "outputs": [],
      "source": [
        "GPU_COUNT = torch.cuda.device_count()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5c3cfade",
      "metadata": {
        "id": "5c3cfade",
        "outputId": "511a715e-cad2-457c-9ed2-b2e4030cedac"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "4"
            ]
          },
          "execution_count": 13,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "GPU_COUNT"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "7bd1e1f2",
      "metadata": {
        "id": "7bd1e1f2",
        "outputId": "1d07efbb-3eb4-48b3-fab2-b8ef364066bd"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "device(type='cuda')"
            ]
          },
          "execution_count": 14,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "device = torch.device(\"cuda\" if torch.cuda.is_available() else \"cpu\") ## specify the GPU id's, GPU id's start from 0.\n",
        "device"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "528023ac",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 177,
          "referenced_widgets": [
            "e6bdda56f1e14d45b1c99940a763e69f",
            "91742ec5cec441b5a94d3046c777b22c",
            "2634942c850f4597be75b3a91f2d75a7",
            "c6727941f53a42248764a106e13596c9",
            "c2f02ef98fc649fca32c157c0d048a89",
            "06f54b167c2e4289b78ba4822d2f98de",
            "9b0a676b668a4ff884a72196524a9229",
            "d76fe4566fb548188ecd8f0575fccb69",
            "394fa40a235147e49fad6edbae132702",
            "521596b5e09b4d8fa79410d3601dcc32",
            "e2258a4aa3ad44d182d19fa6cb7aa05e",
            "998be9daecca4ed6b835333b44a5ba9e",
            "7467ec745de94196b786bf7219744a42",
            "b2d2469f3f0e4639a51ee68b7b74d20a",
            "7eb1948251dd460aa0d42f2d98536ed1",
            "9cec228d653e40e8a3774021542ffb68",
            "81f4f0e1758e4d2498e62dd95c2ea6c6",
            "4db71f00bd814dc3a0e5004eb5a2ae63",
            "802c6b027b704fce99d59f7da5eb1955",
            "df0df21a4ca647e683889a221065b16f",
            "ac83aa99e714487a919ea7aebd0ca422",
            "b963ba7bcd0a481a99d32418a5411ea6",
            "b769f41cc70c4c65a25f0d4a94811bf6",
            "3389a5f07d1a4b4e88c118fe47f241aa",
            "79cac786f31e4ee083b5c637d28eb7fc",
            "ff99ea7820634adb98d061ef265f2ff4",
            "c91b598e790e4c6ba21aeab31c177bf0",
            "a9edca0387954a09ada230067e7355b2",
            "a8b2871e2a164c94a58762d5cd3a2d95",
            "d20681be016d438eb638f1c7108f0d6a",
            "5abf07ae3e3c4f0eb8c12334ed383bcf",
            "58efb1c2ea714ee0a373ecc67d72368f",
            "39801cbd09484e439afb2829be9a9d52",
            "17cd86d84b43442081de52f15f833a71",
            "dbfa65b199f44020be9db0547b7ab18e",
            "d4cb162e77d844d8a672746573950427",
            "f00d00978f0f4077a4d10eeec352734f",
            "94c0b3dad8754852b8f727454c03c814",
            "76ffde2b8d2a4e1ba6a237bda4f7d856",
            "ba7d87dddcaa4ab1b7b0d832b4b3c451",
            "243d9b8fbc0842cfb96b531e9762bb82",
            "7b3448ed1e9442fa92acd84ea146ac2d",
            "1f6c9e48bf54406da669ee35c3f61d11",
            "fb9ddde1db3f47c5b8d7febc0ab691de",
            "ba13e64517a740bc89b29e25dac9de4f",
            "3f87d08da51c42ff8c909b12ca2df863",
            "e7b62de8e4794cd599198ad69e062f2b",
            "0e853c8f845d44fd85b172c8200ec82a",
            "743e49cf0f5a464b99ab5e1b96c81543",
            "fb6b72144db74d8e85489007e0b97396",
            "18c2c03f320e4c59a6451e13d3ce9221",
            "862d52c5e86f49f68013d234d1cb80e1",
            "d2290c10bbff46548e613f5045eb7117",
            "615e44bd9b404f96ab1ca449cb14d30d",
            "a0329c633eb04e0594ed4f033d25bc37"
          ]
        },
        "id": "528023ac",
        "outputId": "deb976ff-2225-45d1-9bce-ced92357067b"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2022-11-03 15:18:28.534264: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2022-11-03 15:18:28.796577: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2022-11-03 15:18:30.642085: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:30.642256: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:30.642272: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
          ]
        }
      ],
      "source": [
        "from transformers import AutoTokenizer, AutoModel\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"cambridgeltl/SapBERT-from-PubMedBERT-fulltext\")\n",
        "model = AutoModel.from_pretrained(\"cambridgeltl/SapBERT-from-PubMedBERT-fulltext\")"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "tlzJasirUq6Y",
      "metadata": {
        "id": "tlzJasirUq6Y",
        "outputId": "6fa6247e-3bb5-4e07-b976-09f3b62a26db"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "2022-11-03 15:18:42.016139: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2022-11-03 15:18:42.048513: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2022-11-03 15:18:42.083803: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2022-11-03 15:18:42.090996: I tensorflow/core/platform/cpu_feature_guard.cc:193] This TensorFlow binary is optimized with oneAPI Deep Neural Network Library (oneDNN) to use the following CPU instructions in performance-critical operations:  AVX2 FMA\n",
            "To enable them in other operations, rebuild TensorFlow with the appropriate compiler flags.\n",
            "2022-11-03 15:18:42.302304: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2022-11-03 15:18:42.302975: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2022-11-03 15:18:42.342529: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2022-11-03 15:18:42.344770: E tensorflow/stream_executor/cuda/cuda_blas.cc:2981] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
            "2022-11-03 15:18:43.882634: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.882867: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.882893: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
            "2022-11-03 15:18:43.895654: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.895895: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.895920: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
            "2022-11-03 15:18:43.897874: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.898080: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.898103: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n",
            "2022-11-03 15:18:43.945918: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer.so.7'; dlerror: libnvinfer.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.946181: W tensorflow/stream_executor/platform/default/dso_loader.cc:64] Could not load dynamic library 'libnvinfer_plugin.so.7'; dlerror: libnvinfer_plugin.so.7: cannot open shared object file: No such file or directory; LD_LIBRARY_PATH: /usr/local/cuda-10.2/lib64:/opt/cudnn-7.6.5.32-cuda-10.2/lib64\n",
            "2022-11-03 15:18:43.946208: W tensorflow/compiler/tf2tensorrt/utils/py_utils.cc:38] TF-TRT Warning: Cannot dlopen some TensorRT libraries. If you would like to use Nvidia GPU with TensorRT, please make sure the missing libraries mentioned above are installed properly.\n"
          ]
        },
        {
          "data": {
            "text/plain": [
              "<parallelformers.parallelize.parallelize at 0x14a2954f9f40>"
            ]
          },
          "execution_count": 16,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "# model = torch.nn.DataParallel(model)\n",
        "# model.to(device)\n",
        "from parallelformers import parallelize\n",
        "parallelize(model, num_gpus=4, fp16=True)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "a3a24048",
      "metadata": {
        "id": "a3a24048"
      },
      "source": [
        "Generate embeddings for snomed labels"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "bb0b8655",
      "metadata": {
        "id": "bb0b8655"
      },
      "outputs": [],
      "source": [
        "# all_names1 = all_names[:100]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5c5ff31c",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5c5ff31c",
        "outputId": "b229214d-80aa-4d6d-8ca7-7325c60944d6"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|█████████████████████████████████████| 12260/12260 [17:08<00:00, 11.92it/s]\n"
          ]
        }
      ],
      "source": [
        "bs = 128\n",
        "all_reps = []\n",
        "for i in tqdm(np.arange(0, len(all_names), bs)):\n",
        "    toks = tokenizer.batch_encode_plus(all_names[i:i+bs],\n",
        "                                      padding=\"max_length\",\n",
        "                                      max_length=25,\n",
        "                                      truncation=True,\n",
        "                                      return_tensors=\"pt\")\n",
        "    toks = toks.to(device)\n",
        "    output = model(**toks)\n",
        "    cls_rep = output[0][:,0,:]\n",
        "    \n",
        "    all_reps.append(cls_rep.cpu().detach().numpy())\n",
        "all_reps_emb = np.concatenate(all_reps, axis=0)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "c1230654",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "c1230654",
        "outputId": "a552bfff-4e03-4ac4-e3c4-34365c66c708"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "(1569232, 768)\n"
          ]
        }
      ],
      "source": [
        "print(all_reps_emb.shape)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "00a427c1",
      "metadata": {
        "id": "00a427c1"
      },
      "outputs": [],
      "source": [
        "all_reps_emb = all_reps_emb.astype(np.float32)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3c884582",
      "metadata": {
        "id": "3c884582"
      },
      "outputs": [],
      "source": [
        "import faiss"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "9d7d069d",
      "metadata": {
        "id": "9d7d069d",
        "outputId": "ff1b1498-4628-4c4a-edda-d98637fcace7"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "True\n"
          ]
        }
      ],
      "source": [
        "d = all_reps_emb.shape[1]\n",
        "index = faiss.IndexFlatL2(d)   # build the index\n",
        "print(index.is_trained)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "77b258e0",
      "metadata": {
        "id": "77b258e0",
        "outputId": "1bde8c9e-b1ed-420a-8bea-fc81b2090075"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "1569232\n"
          ]
        }
      ],
      "source": [
        "index.add(all_reps_emb)                  # add vectors to the index\n",
        "print(index.ntotal)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "40fe39a4",
      "metadata": {
        "id": "40fe39a4"
      },
      "source": [
        "Load ground truth data"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "44851e30",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 337
        },
        "id": "44851e30",
        "outputId": "652b992e-de38-468a-e2f3-e5b48a60c143"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>filename</th>\n",
              "      <th>mark</th>\n",
              "      <th>label</th>\n",
              "      <th>offset1</th>\n",
              "      <th>offset2</th>\n",
              "      <th>span</th>\n",
              "      <th>code</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>es-S0212-71992007000100007-1</td>\n",
              "      <td>T1</td>\n",
              "      <td>ENFERMEDAD</td>\n",
              "      <td>40</td>\n",
              "      <td>61</td>\n",
              "      <td>arterial hypertension</td>\n",
              "      <td>38341003</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>es-S0212-71992007000100007-1</td>\n",
              "      <td>T2</td>\n",
              "      <td>ENFERMEDAD</td>\n",
              "      <td>66</td>\n",
              "      <td>79</td>\n",
              "      <td>polyarthrosis</td>\n",
              "      <td>36186002</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>es-S0212-71992007000100007-1</td>\n",
              "      <td>T3</td>\n",
              "      <td>ENFERMEDAD</td>\n",
              "      <td>1682</td>\n",
              "      <td>1698</td>\n",
              "      <td>pleural effusion</td>\n",
              "      <td>60046008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>es-S0212-71992007000100007-1</td>\n",
              "      <td>T4</td>\n",
              "      <td>ENFERMEDAD</td>\n",
              "      <td>1859</td>\n",
              "      <td>1875</td>\n",
              "      <td>pleural effusion</td>\n",
              "      <td>60046008</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>es-S0212-71992007000100007-1</td>\n",
              "      <td>T5</td>\n",
              "      <td>ENFERMEDAD</td>\n",
              "      <td>1626</td>\n",
              "      <td>1648</td>\n",
              "      <td>lower lobe atelectasis</td>\n",
              "      <td>46621007</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                       filename mark       label  offset1  offset2  \\\n",
              "0  es-S0212-71992007000100007-1   T1  ENFERMEDAD       40       61   \n",
              "1  es-S0212-71992007000100007-1   T2  ENFERMEDAD       66       79   \n",
              "2  es-S0212-71992007000100007-1   T3  ENFERMEDAD     1682     1698   \n",
              "3  es-S0212-71992007000100007-1   T4  ENFERMEDAD     1859     1875   \n",
              "4  es-S0212-71992007000100007-1   T5  ENFERMEDAD     1626     1648   \n",
              "\n",
              "                     span      code  \n",
              "0   arterial hypertension  38341003  \n",
              "1           polyarthrosis  36186002  \n",
              "2        pleural effusion  60046008  \n",
              "3        pleural effusion  60046008  \n",
              "4  lower lobe atelectasis  46621007  "
            ]
          },
          "execution_count": 27,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "entities = pd.read_csv(\"data/entities.tsv\", delimiter=\"\\t\")\n",
        "entities.head()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "8a009c68",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "8a009c68",
        "outputId": "0b1bfb0f-0fa5-4487-d95c-2723a0c82f00",
        "scrolled": true
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "['arterial hypertension', 'polyarthrosis', 'pleural effusion', 'pleural effusion', 'lower lobe atelectasis', 'infectious spondylodiscitis d10-d11', 'pleural effusion', 'brucellosis', 'orchiepididymitis', 'goitre']\n",
            "0     38341003\n",
            "1     36186002\n",
            "2     60046008\n",
            "3     60046008\n",
            "4     46621007\n",
            "5    302935008\n",
            "6     60046008\n",
            "7     75702008\n",
            "8    197983000\n",
            "9      3716002\n",
            "Name: code, dtype: object\n"
          ]
        }
      ],
      "source": [
        "inp_names = [i.lower() for i in entities['span']]\n",
        "inp_labels = entities['code']\n",
        "print(inp_names[:10])\n",
        "print(inp_labels[:10])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "90bbf268",
      "metadata": {
        "id": "90bbf268"
      },
      "outputs": [],
      "source": [
        "# c=0\n",
        "# for i in inp_label:\n",
        "# #     if type(i)!=float:\n",
        "#     try:\n",
        "#         [float(i)]\n",
        "#     except:\n",
        "#         c+=1\n",
        "# #         print(i.split('+'))\n",
        "# c"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "49562d03",
      "metadata": {
        "id": "49562d03"
      },
      "outputs": [],
      "source": [
        "# inp_names1 = inp_names[:10]"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "e6cf5d29",
      "metadata": {
        "id": "e6cf5d29"
      },
      "source": [
        "Generate embeddings for ground truth terms, get their closest snomedct embedding and list out its corresponding snomedct code"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "049818b3",
      "metadata": {
        "id": "049818b3"
      },
      "outputs": [],
      "source": [
        "query_toks = tokenizer.batch_encode_plus(list(inp_names),\n",
        "                                        padding = \"max_length\",\n",
        "                                        max_length = 25,\n",
        "                                        truncation=True,\n",
        "                                        return_tensors=\"pt\")\n",
        "query_toks = query_toks.to(device)\n",
        "query_output = model(**query_toks)\n",
        "query_cls_rep = query_output[0][:,0,:]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "f0ab19b8",
      "metadata": {
        "id": "f0ab19b8"
      },
      "outputs": [],
      "source": [
        "query_cls_rep = query_cls_rep.cpu().detach().numpy()"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "3d90a519",
      "metadata": {
        "id": "3d90a519"
      },
      "outputs": [],
      "source": [
        "query_cls_rep = query_cls_rep.astype(np.float32)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "184cd570",
      "metadata": {
        "id": "184cd570"
      },
      "outputs": [],
      "source": [
        "k= 1 # take the 1 closest neighbor"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ac0965a5",
      "metadata": {
        "id": "ac0965a5"
      },
      "outputs": [],
      "source": [
        "D, I = index.search(query_cls_rep, k)"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "e7fada1e",
      "metadata": {
        "id": "e7fada1e",
        "outputId": "3da12aec-6ca8-42ed-84ef-bb0a1f7589b0"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "[[473409]\n",
            " [ 58583]\n",
            " [ 96888]\n",
            " [ 96888]\n",
            " [477684]]\n"
          ]
        }
      ],
      "source": [
        "print(I[:5])"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "4cfb69ca",
      "metadata": {
        "id": "4cfb69ca"
      },
      "outputs": [],
      "source": []
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "2145e65a",
      "metadata": {
        "id": "2145e65a"
      },
      "outputs": [],
      "source": [
        "pred_ids = [all_ids[i[0]] for i in I]\n",
        "# score=sum((pred_ids[i]==inp_label[i])*1 for i in range(len(pred_ids)))\n",
        "# score/len(inp_label)"
      ]
    },
    {
      "cell_type": "markdown",
      "id": "85c1243b",
      "metadata": {
        "id": "85c1243b"
      },
      "source": [
        "In ground truth, zero or more than one codes are also present for each term; here only one code is predicted for each query"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "d7476a77",
      "metadata": {
        "id": "d7476a77"
      },
      "outputs": [],
      "source": [
        "p = [[i] for i in pred_ids]\n",
        "t = []\n",
        "for i in inp_labels:\n",
        "    try:\n",
        "        t.append([int(i)])\n",
        "    except:\n",
        "        try:\n",
        "            t.append([int(j) for j in (i.split('+'))])\n",
        "        except:\n",
        "#             print('nomap')\n",
        "            t.append([])\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "5a676132",
      "metadata": {
        "id": "5a676132",
        "outputId": "2f641e72-e210-4828-965e-03a19f935586"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "38341003"
            ]
          },
          "execution_count": 49,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "p[0][0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "424b2281",
      "metadata": {
        "id": "424b2281",
        "outputId": "59a92896-28b7-49f2-e3c8-867d987a78c2"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "execution_count": 50,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "p[0][0] in t[0]"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "id": "ae535967",
      "metadata": {
        "id": "ae535967",
        "outputId": "bd4df60d-5218-40bf-81a5-2cb7e7150e5d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "precision 0.3675187969924812\n",
            "recall 0.359147685525349\n",
            "f1 0.3632849645578643\n"
          ]
        }
      ],
      "source": [
        "pre = 0\n",
        "for i in range(len(p)):\n",
        "    if p[i][0] in t[i]:\n",
        "        pre+=1\n",
        "\n",
        "pre /= len(p)\n",
        "print('precision', pre)\n",
        "\n",
        "\n",
        "rec = 0\n",
        "for i in range(len(t)):\n",
        "    if len(t[i])==1:\n",
        "        if t[i][0] in p[i]:\n",
        "            rec+=1\n",
        "    elif len(t[i])>1:\n",
        "        for j in range(len(t[i])):\n",
        "            if t[i][j] in p[i]:\n",
        "                rec+=1\n",
        "\n",
        "rec /= sum(len(i) for i in t)\n",
        "print('recall', rec)       \n",
        "\n",
        "\n",
        "f1 = 2*pre*rec/(pre+rec+np.finfo(np.float32).eps)\n",
        "print('f1', f1)"
      ]
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "machine_shape": "hm",
      "provenance": []
    },
    "gpuClass": "premium",
    "kernelspec": {
      "display_name": "Python 3 (ipykernel)",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "06f54b167c2e4289b78ba4822d2f98de": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "0e853c8f845d44fd85b172c8200ec82a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_615e44bd9b404f96ab1ca449cb14d30d",
            "placeholder": "​",
            "style": "IPY_MODEL_a0329c633eb04e0594ed4f033d25bc37",
            "value": " 438M/438M [00:12&lt;00:00, 34.9MB/s]"
          }
        },
        "17cd86d84b43442081de52f15f833a71": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_dbfa65b199f44020be9db0547b7ab18e",
              "IPY_MODEL_d4cb162e77d844d8a672746573950427",
              "IPY_MODEL_f00d00978f0f4077a4d10eeec352734f"
            ],
            "layout": "IPY_MODEL_94c0b3dad8754852b8f727454c03c814"
          }
        },
        "18c2c03f320e4c59a6451e13d3ce9221": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "1f6c9e48bf54406da669ee35c3f61d11": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "243d9b8fbc0842cfb96b531e9762bb82": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "2634942c850f4597be75b3a91f2d75a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d76fe4566fb548188ecd8f0575fccb69",
            "max": 198,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_394fa40a235147e49fad6edbae132702",
            "value": 198
          }
        },
        "3389a5f07d1a4b4e88c118fe47f241aa": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_a9edca0387954a09ada230067e7355b2",
            "placeholder": "​",
            "style": "IPY_MODEL_a8b2871e2a164c94a58762d5cd3a2d95",
            "value": "Downloading: 100%"
          }
        },
        "394fa40a235147e49fad6edbae132702": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "39801cbd09484e439afb2829be9a9d52": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "3f87d08da51c42ff8c909b12ca2df863": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_fb6b72144db74d8e85489007e0b97396",
            "placeholder": "​",
            "style": "IPY_MODEL_18c2c03f320e4c59a6451e13d3ce9221",
            "value": "Downloading: 100%"
          }
        },
        "4db71f00bd814dc3a0e5004eb5a2ae63": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "521596b5e09b4d8fa79410d3601dcc32": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "58efb1c2ea714ee0a373ecc67d72368f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5abf07ae3e3c4f0eb8c12334ed383bcf": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "615e44bd9b404f96ab1ca449cb14d30d": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "743e49cf0f5a464b99ab5e1b96c81543": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7467ec745de94196b786bf7219744a42": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_81f4f0e1758e4d2498e62dd95c2ea6c6",
            "placeholder": "​",
            "style": "IPY_MODEL_4db71f00bd814dc3a0e5004eb5a2ae63",
            "value": "Downloading: 100%"
          }
        },
        "76ffde2b8d2a4e1ba6a237bda4f7d856": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "79cac786f31e4ee083b5c637d28eb7fc": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_d20681be016d438eb638f1c7108f0d6a",
            "max": 226150,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5abf07ae3e3c4f0eb8c12334ed383bcf",
            "value": 226150
          }
        },
        "7b3448ed1e9442fa92acd84ea146ac2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "7eb1948251dd460aa0d42f2d98536ed1": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_ac83aa99e714487a919ea7aebd0ca422",
            "placeholder": "​",
            "style": "IPY_MODEL_b963ba7bcd0a481a99d32418a5411ea6",
            "value": " 462/462 [00:00&lt;00:00, 11.0kB/s]"
          }
        },
        "802c6b027b704fce99d59f7da5eb1955": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "81f4f0e1758e4d2498e62dd95c2ea6c6": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "862d52c5e86f49f68013d234d1cb80e1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "91742ec5cec441b5a94d3046c777b22c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_06f54b167c2e4289b78ba4822d2f98de",
            "placeholder": "​",
            "style": "IPY_MODEL_9b0a676b668a4ff884a72196524a9229",
            "value": "Downloading: 100%"
          }
        },
        "94c0b3dad8754852b8f727454c03c814": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "998be9daecca4ed6b835333b44a5ba9e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_7467ec745de94196b786bf7219744a42",
              "IPY_MODEL_b2d2469f3f0e4639a51ee68b7b74d20a",
              "IPY_MODEL_7eb1948251dd460aa0d42f2d98536ed1"
            ],
            "layout": "IPY_MODEL_9cec228d653e40e8a3774021542ffb68"
          }
        },
        "9b0a676b668a4ff884a72196524a9229": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "9cec228d653e40e8a3774021542ffb68": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "a0329c633eb04e0594ed4f033d25bc37": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a8b2871e2a164c94a58762d5cd3a2d95": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "a9edca0387954a09ada230067e7355b2": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ac83aa99e714487a919ea7aebd0ca422": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b2d2469f3f0e4639a51ee68b7b74d20a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_802c6b027b704fce99d59f7da5eb1955",
            "max": 462,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_df0df21a4ca647e683889a221065b16f",
            "value": 462
          }
        },
        "b769f41cc70c4c65a25f0d4a94811bf6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3389a5f07d1a4b4e88c118fe47f241aa",
              "IPY_MODEL_79cac786f31e4ee083b5c637d28eb7fc",
              "IPY_MODEL_ff99ea7820634adb98d061ef265f2ff4"
            ],
            "layout": "IPY_MODEL_c91b598e790e4c6ba21aeab31c177bf0"
          }
        },
        "b963ba7bcd0a481a99d32418a5411ea6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ba13e64517a740bc89b29e25dac9de4f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_3f87d08da51c42ff8c909b12ca2df863",
              "IPY_MODEL_e7b62de8e4794cd599198ad69e062f2b",
              "IPY_MODEL_0e853c8f845d44fd85b172c8200ec82a"
            ],
            "layout": "IPY_MODEL_743e49cf0f5a464b99ab5e1b96c81543"
          }
        },
        "ba7d87dddcaa4ab1b7b0d832b4b3c451": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "c2f02ef98fc649fca32c157c0d048a89": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c6727941f53a42248764a106e13596c9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_521596b5e09b4d8fa79410d3601dcc32",
            "placeholder": "​",
            "style": "IPY_MODEL_e2258a4aa3ad44d182d19fa6cb7aa05e",
            "value": " 198/198 [00:00&lt;00:00, 5.39kB/s]"
          }
        },
        "c91b598e790e4c6ba21aeab31c177bf0": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d20681be016d438eb638f1c7108f0d6a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d2290c10bbff46548e613f5045eb7117": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d4cb162e77d844d8a672746573950427": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_243d9b8fbc0842cfb96b531e9762bb82",
            "max": 112,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_7b3448ed1e9442fa92acd84ea146ac2d",
            "value": 112
          }
        },
        "d76fe4566fb548188ecd8f0575fccb69": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "dbfa65b199f44020be9db0547b7ab18e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_76ffde2b8d2a4e1ba6a237bda4f7d856",
            "placeholder": "​",
            "style": "IPY_MODEL_ba7d87dddcaa4ab1b7b0d832b4b3c451",
            "value": "Downloading: 100%"
          }
        },
        "df0df21a4ca647e683889a221065b16f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "e2258a4aa3ad44d182d19fa6cb7aa05e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e6bdda56f1e14d45b1c99940a763e69f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_91742ec5cec441b5a94d3046c777b22c",
              "IPY_MODEL_2634942c850f4597be75b3a91f2d75a7",
              "IPY_MODEL_c6727941f53a42248764a106e13596c9"
            ],
            "layout": "IPY_MODEL_c2f02ef98fc649fca32c157c0d048a89"
          }
        },
        "e7b62de8e4794cd599198ad69e062f2b": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_862d52c5e86f49f68013d234d1cb80e1",
            "max": 438012727,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d2290c10bbff46548e613f5045eb7117",
            "value": 438012727
          }
        },
        "f00d00978f0f4077a4d10eeec352734f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_1f6c9e48bf54406da669ee35c3f61d11",
            "placeholder": "​",
            "style": "IPY_MODEL_fb9ddde1db3f47c5b8d7febc0ab691de",
            "value": " 112/112 [00:00&lt;00:00, 4.52kB/s]"
          }
        },
        "fb6b72144db74d8e85489007e0b97396": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "fb9ddde1db3f47c5b8d7febc0ab691de": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "ff99ea7820634adb98d061ef265f2ff4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_58efb1c2ea714ee0a373ecc67d72368f",
            "placeholder": "​",
            "style": "IPY_MODEL_39801cbd09484e439afb2829be9a9d52",
            "value": " 226k/226k [00:00&lt;00:00, 1.73MB/s]"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}