InChI_work_notebook.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/phelchegs/bms-molecular-translation/blob/main/InChI_work_notebook.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ehGcgt1UQ6f1",
        "outputId": "7a663e1e-351a-4c7a-a69b-d410d2d2d186"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting rdkit\n",
            "  Downloading rdkit-2023.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (29.7 MB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m29.7/29.7 MB\u001b[0m \u001b[31m21.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hRequirement already satisfied: numpy in /usr/local/lib/python3.10/dist-packages (from rdkit) (1.23.5)\n",
            "Requirement already satisfied: Pillow in /usr/local/lib/python3.10/dist-packages (from rdkit) (9.4.0)\n",
            "Installing collected packages: rdkit\n",
            "Successfully installed rdkit-2023.3.3\n",
            "Collecting einops\n",
            "  Downloading einops-0.6.1-py3-none-any.whl (42 kB)\n",
            "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m42.2/42.2 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25hInstalling collected packages: einops\n",
            "Successfully installed einops-0.6.1\n",
            "Collecting GPUtil\n",
            "  Downloading GPUtil-1.4.0.tar.gz (5.5 kB)\n",
            "  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: GPUtil\n",
            "  Building wheel for GPUtil (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for GPUtil: filename=GPUtil-1.4.0-py3-none-any.whl size=7392 sha256=620bf7e81b60ecbf9afd7382e5941ce5e15e97cf1339a40f71d6c0b3df07f73a\n",
            "  Stored in directory: /root/.cache/pip/wheels/a9/8a/bd/81082387151853ab8b6b3ef33426e98f5cbfebc3c397a9d4d0\n",
            "Successfully built GPUtil\n",
            "Installing collected packages: GPUtil\n",
            "Successfully installed GPUtil-1.4.0\n"
          ]
        }
      ],
      "source": [
        "#install all necessary packages. RDKit is used for pics of mol structures for extra dataset.\n",
        "#\n",
        "!pip install rdkit\n",
        "!pip install einops\n",
        "!pip install GPUtil\n",
        "!pip install -q kaggle"
      ],
      "id": "ehGcgt1UQ6f1"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 90
        },
        "id": "ksW_npoRTRq2",
        "outputId": "820f93e7-1b23-445c-f2a7-e8c5fc6443e9"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "     <input type=\"file\" id=\"files-b722df78-754e-4ec1-b978-0b6d58b9432d\" name=\"files[]\" multiple disabled\n",
              "        style=\"border:none\" />\n",
              "     <output id=\"result-b722df78-754e-4ec1-b978-0b6d58b9432d\">\n",
              "      Upload widget is only available when the cell has been executed in the\n",
              "      current browser session. Please rerun this cell to enable.\n",
              "      </output>\n",
              "      <script>// Copyright 2017 Google LLC\n",
              "//\n",
              "// Licensed under the Apache License, Version 2.0 (the \"License\");\n",
              "// you may not use this file except in compliance with the License.\n",
              "// You may obtain a copy of the License at\n",
              "//\n",
              "//      http://www.apache.org/licenses/LICENSE-2.0\n",
              "//\n",
              "// Unless required by applicable law or agreed to in writing, software\n",
              "// distributed under the License is distributed on an \"AS IS\" BASIS,\n",
              "// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n",
              "// See the License for the specific language governing permissions and\n",
              "// limitations under the License.\n",
              "\n",
              "/**\n",
              " * @fileoverview Helpers for google.colab Python module.\n",
              " */\n",
              "(function(scope) {\n",
              "function span(text, styleAttributes = {}) {\n",
              "  const element = document.createElement('span');\n",
              "  element.textContent = text;\n",
              "  for (const key of Object.keys(styleAttributes)) {\n",
              "    element.style[key] = styleAttributes[key];\n",
              "  }\n",
              "  return element;\n",
              "}\n",
              "\n",
              "// Max number of bytes which will be uploaded at a time.\n",
              "const MAX_PAYLOAD_SIZE = 100 * 1024;\n",
              "\n",
              "function _uploadFiles(inputId, outputId) {\n",
              "  const steps = uploadFilesStep(inputId, outputId);\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  // Cache steps on the outputElement to make it available for the next call\n",
              "  // to uploadFilesContinue from Python.\n",
              "  outputElement.steps = steps;\n",
              "\n",
              "  return _uploadFilesContinue(outputId);\n",
              "}\n",
              "\n",
              "// This is roughly an async generator (not supported in the browser yet),\n",
              "// where there are multiple asynchronous steps and the Python side is going\n",
              "// to poll for completion of each step.\n",
              "// This uses a Promise to block the python side on completion of each step,\n",
              "// then passes the result of the previous step as the input to the next step.\n",
              "function _uploadFilesContinue(outputId) {\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  const steps = outputElement.steps;\n",
              "\n",
              "  const next = steps.next(outputElement.lastPromiseValue);\n",
              "  return Promise.resolve(next.value.promise).then((value) => {\n",
              "    // Cache the last promise value to make it available to the next\n",
              "    // step of the generator.\n",
              "    outputElement.lastPromiseValue = value;\n",
              "    return next.value.response;\n",
              "  });\n",
              "}\n",
              "\n",
              "/**\n",
              " * Generator function which is called between each async step of the upload\n",
              " * process.\n",
              " * @param {string} inputId Element ID of the input file picker element.\n",
              " * @param {string} outputId Element ID of the output display.\n",
              " * @return {!Iterable<!Object>} Iterable of next steps.\n",
              " */\n",
              "function* uploadFilesStep(inputId, outputId) {\n",
              "  const inputElement = document.getElementById(inputId);\n",
              "  inputElement.disabled = false;\n",
              "\n",
              "  const outputElement = document.getElementById(outputId);\n",
              "  outputElement.innerHTML = '';\n",
              "\n",
              "  const pickedPromise = new Promise((resolve) => {\n",
              "    inputElement.addEventListener('change', (e) => {\n",
              "      resolve(e.target.files);\n",
              "    });\n",
              "  });\n",
              "\n",
              "  const cancel = document.createElement('button');\n",
              "  inputElement.parentElement.appendChild(cancel);\n",
              "  cancel.textContent = 'Cancel upload';\n",
              "  const cancelPromise = new Promise((resolve) => {\n",
              "    cancel.onclick = () => {\n",
              "      resolve(null);\n",
              "    };\n",
              "  });\n",
              "\n",
              "  // Wait for the user to pick the files.\n",
              "  const files = yield {\n",
              "    promise: Promise.race([pickedPromise, cancelPromise]),\n",
              "    response: {\n",
              "      action: 'starting',\n",
              "    }\n",
              "  };\n",
              "\n",
              "  cancel.remove();\n",
              "\n",
              "  // Disable the input element since further picks are not allowed.\n",
              "  inputElement.disabled = true;\n",
              "\n",
              "  if (!files) {\n",
              "    return {\n",
              "      response: {\n",
              "        action: 'complete',\n",
              "      }\n",
              "    };\n",
              "  }\n",
              "\n",
              "  for (const file of files) {\n",
              "    const li = document.createElement('li');\n",
              "    li.append(span(file.name, {fontWeight: 'bold'}));\n",
              "    li.append(span(\n",
              "        `(${file.type || 'n/a'}) - ${file.size} bytes, ` +\n",
              "        `last modified: ${\n",
              "            file.lastModifiedDate ? file.lastModifiedDate.toLocaleDateString() :\n",
              "                                    'n/a'} - `));\n",
              "    const percent = span('0% done');\n",
              "    li.appendChild(percent);\n",
              "\n",
              "    outputElement.appendChild(li);\n",
              "\n",
              "    const fileDataPromise = new Promise((resolve) => {\n",
              "      const reader = new FileReader();\n",
              "      reader.onload = (e) => {\n",
              "        resolve(e.target.result);\n",
              "      };\n",
              "      reader.readAsArrayBuffer(file);\n",
              "    });\n",
              "    // Wait for the data to be ready.\n",
              "    let fileData = yield {\n",
              "      promise: fileDataPromise,\n",
              "      response: {\n",
              "        action: 'continue',\n",
              "      }\n",
              "    };\n",
              "\n",
              "    // Use a chunked sending to avoid message size limits. See b/62115660.\n",
              "    let position = 0;\n",
              "    do {\n",
              "      const length = Math.min(fileData.byteLength - position, MAX_PAYLOAD_SIZE);\n",
              "      const chunk = new Uint8Array(fileData, position, length);\n",
              "      position += length;\n",
              "\n",
              "      const base64 = btoa(String.fromCharCode.apply(null, chunk));\n",
              "      yield {\n",
              "        response: {\n",
              "          action: 'append',\n",
              "          file: file.name,\n",
              "          data: base64,\n",
              "        },\n",
              "      };\n",
              "\n",
              "      let percentDone = fileData.byteLength === 0 ?\n",
              "          100 :\n",
              "          Math.round((position / fileData.byteLength) * 100);\n",
              "      percent.textContent = `${percentDone}% done`;\n",
              "\n",
              "    } while (position < fileData.byteLength);\n",
              "  }\n",
              "\n",
              "  // All done.\n",
              "  yield {\n",
              "    response: {\n",
              "      action: 'complete',\n",
              "    }\n",
              "  };\n",
              "}\n",
              "\n",
              "scope.google = scope.google || {};\n",
              "scope.google.colab = scope.google.colab || {};\n",
              "scope.google.colab._files = {\n",
              "  _uploadFiles,\n",
              "  _uploadFilesContinue,\n",
              "};\n",
              "})(self);\n",
              "</script> "
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saving kaggle.json to kaggle.json\n",
            "User uploaded file \"kaggle.json\" with length 68 bytes\n"
          ]
        }
      ],
      "source": [
        "#import kaggle beta API to download the data to 'Files' folder.\n",
        "\n",
        "from google.colab import files\n",
        "\n",
        "uploaded = files.upload()\n",
        "\n",
        "for fn in uploaded.keys():\n",
        "  print('User uploaded file \"{name}\" with length {length} bytes'.format(\n",
        "      name=fn, length=len(uploaded[fn])))\n",
        "\n",
        "# Then move kaggle.json into the folder where the API expects to find it.\n",
        "!mkdir -p ~/.kaggle/ && mv kaggle.json ~/.kaggle/ && chmod 600 ~/.kaggle/kaggle.json"
      ],
      "id": "ksW_npoRTRq2"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "R4Itpa8xTZ_c",
        "outputId": "d9db3868-66ca-4ea3-99cb-ca71a4550ecd"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Collecting kaggle\n",
            "  Downloading kaggle-1.5.16.tar.gz (83 kB)\n",
            "\u001b[?25l     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m0.0/83.6 kB\u001b[0m \u001b[31m?\u001b[0m eta \u001b[36m-:--:--\u001b[0m\r\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m83.6/83.6 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
            "\u001b[?25h  Preparing metadata (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "Building wheels for collected packages: kaggle\n",
            "  Building wheel for kaggle (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for kaggle: filename=kaggle-1.5.16-py3-none-any.whl size=110683 sha256=bf3f2cb4702eaf4a3cb694e07330b255529be2ee8832acb80f32823c431620ee\n",
            "  Stored in directory: /root/.cache/pip/wheels/43/4b/fb/736478af5e8004810081a06259f9aa2f7c3329fc5d03c2c412\n",
            "Successfully built kaggle\n",
            "Installing collected packages: kaggle\n",
            "  Attempting uninstall: kaggle\n",
            "    Found existing installation: kaggle 1.5.16\n",
            "    Uninstalling kaggle-1.5.16:\n",
            "      Successfully uninstalled kaggle-1.5.16\n",
            "Successfully installed kaggle-1.5.16\n"
          ]
        }
      ],
      "source": [
        "!pip install --upgrade --force-reinstall --no-deps kaggle"
      ],
      "id": "R4Itpa8xTZ_c"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "MhhtIKKnTenf",
        "outputId": "65f1f6e3-9496-4996-f80e-e5d7c69a05a4"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Downloading bms-molecular-translation.zip to /content\n",
            "100% 7.81G/7.82G [01:18<00:00, 81.0MB/s]\n",
            "100% 7.82G/7.82G [01:18<00:00, 106MB/s] \n"
          ]
        }
      ],
      "source": [
        "#Dowwnload.\n",
        "!kaggle competitions download -c 'bms-molecular-translation'"
      ],
      "id": "MhhtIKKnTenf"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "CPDe1JElTqP5"
      },
      "outputs": [],
      "source": [
        "#Unzip. It will take a long time.\n",
        "!mkdir bms-molecular-translation\n",
        "!unzip bms-molecular-translation.zip -d bms-molecular-translation"
      ],
      "id": "CPDe1JElTqP5"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "o9IonlIphTIK",
        "outputId": "32edeb31-2343-470d-9071-b5aac702e892"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Your runtime has 54.8 gigabytes of available RAM\n",
            "\n",
            "You are using a high-RAM runtime!\n"
          ]
        }
      ],
      "source": [
        "#Show the high-RAM property.\n",
        "from psutil import virtual_memory\n",
        "ram_gb = virtual_memory().total / 1e9\n",
        "print('Your runtime has {:.1f} gigabytes of available RAM\\n'.format(ram_gb))\n",
        "\n",
        "if ram_gb < 20:\n",
        "  print('Not using a high-RAM runtime')\n",
        "else:\n",
        "  print('You are using a high-RAM runtime!')"
      ],
      "id": "o9IonlIphTIK"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ZXKDrdAMg0la",
        "outputId": "4d4dded5-3339-4724-8832-bcaf5116734b"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Fri Sep 22 13:58:50 2023       \n",
            "+-----------------------------------------------------------------------------+\n",
            "| NVIDIA-SMI 525.105.17   Driver Version: 525.105.17   CUDA Version: 12.0     |\n",
            "|-------------------------------+----------------------+----------------------+\n",
            "| GPU  Name        Persistence-M| Bus-Id        Disp.A | Volatile Uncorr. ECC |\n",
            "| Fan  Temp  Perf  Pwr:Usage/Cap|         Memory-Usage | GPU-Util  Compute M. |\n",
            "|                               |                      |               MIG M. |\n",
            "|===============================+======================+======================|\n",
            "|   0  Tesla T4            Off  | 00000000:00:04.0 Off |                    0 |\n",
            "| N/A   36C    P8     9W /  70W |      0MiB / 15360MiB |      0%      Default |\n",
            "|                               |                      |                  N/A |\n",
            "+-------------------------------+----------------------+----------------------+\n",
            "                                                                               \n",
            "+-----------------------------------------------------------------------------+\n",
            "| Processes:                                                                  |\n",
            "|  GPU   GI   CI        PID   Type   Process name                  GPU Memory |\n",
            "|        ID   ID                                                   Usage      |\n",
            "|=============================================================================|\n",
            "|  No running processes found                                                 |\n",
            "+-----------------------------------------------------------------------------+\n"
          ]
        }
      ],
      "source": [
        "#Show CUDA and GPU.\n",
        "gpu_info = !nvidia-smi\n",
        "gpu_info = '\\n'.join(gpu_info)\n",
        "if gpu_info.find('failed') >= 0:\n",
        "  print('Not connected to a GPU')\n",
        "else:\n",
        "  print(gpu_info)"
      ],
      "id": "ZXKDrdAMg0la"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "z5atKduIRzol",
        "outputId": "70268443-fdbf-4c01-a4eb-937994787079"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Mounted at /content/drive\n"
          ]
        }
      ],
      "source": [
        "#Mount the path to content/drive\n",
        "#sys.path.append allows us to call the .py files in drive/MyDrive.\n",
        "import os\n",
        "import sys\n",
        "from google.colab import drive\n",
        "\n",
        "drive.mount(\"/content/drive\")\n",
        "sys.path.append(\"/content/drive/MyDrive/deep learning project-chemical recognition\")"
      ],
      "id": "z5atKduIRzol"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "86e8e326-2554-44c2-9d8b-4a055e6d8474"
      },
      "outputs": [],
      "source": [
        "#Import packages, .py files is used for the transformer model and the InChI dataset preprocessing.\n",
        "import os\n",
        "import re\n",
        "import pickle\n",
        "import numpy as np\n",
        "import pandas as pd\n",
        "from tqdm.auto import tqdm\n",
        "tqdm.pandas()\n",
        "import torch\n",
        "from torch.utils.data import Dataset\n",
        "from sklearn.model_selection import KFold\n",
        "\n",
        "from InChI_tokenizer import Tokenizer\n",
        "from InChI_preprocessing import split_form, train_file_path, get_atom_counts, crop_image, pad_image, preprocess_train_images, get_transform, get_aug\n",
        "from InChI_dataset import Extra_inchi_save_ds, MolecularTranslationDataModule\n",
        "from InChI_tokenizer import Tokenizer\n",
        "from utils_InChI_translate import shape_after_crop, run_one_epoch, DummyOptimizer, DummyScheduler, Train_Counter\n",
        "\n",
        "import cv2\n",
        "import rdkit.Chem as Chem\n",
        "\n",
        "from matplotlib import pyplot as plt\n",
        "%matplotlib inline"
      ],
      "id": "86e8e326-2554-44c2-9d8b-4a055e6d8474"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "0LZkthD9-ssf"
      },
      "outputs": [],
      "source": [
        "from Transformer_decoder import subsequent_mask\n",
        "from Transformer_embedding import TGT_Embedding\n",
        "from cnn_ViT_transformer import ViT_transformer\n",
        "import utils\n",
        "import MNIST_example\n",
        "import altair as alt\n",
        "import copy\n",
        "import math\n",
        "from torch import nn\n",
        "from torch.optim.lr_scheduler import LambdaLR\n",
        "import matplotlib.pyplot as plt\n",
        "import GPUtil"
      ],
      "id": "0LZkthD9-ssf"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "cb4e8eb0-68ab-4000-baa0-17dcc75bae1d"
      },
      "outputs": [],
      "source": [
        "#Define the configuration class. Note that the max_text_len and vocab_length are obtained from the later cells, which means this cell needs to rerun after obtaining those 2 parameters.\n",
        "\n",
        "class CFG:\n",
        "    #environment\n",
        "    device = 0\n",
        "\n",
        "    #general\n",
        "    seed = 1987\n",
        "\n",
        "    #dataset\n",
        "    batch_size = 64\n",
        "\n",
        "    #preprocess\n",
        "    trans_type = 'rotate90 or verticalflip'\n",
        "    pixels_mean = (0.485)\n",
        "    pixels_std = (0.229)\n",
        "    crop = True\n",
        "\n",
        "    #key parameter\n",
        "    #max_crop_size = 1485\n",
        "    max_text_len = 268\n",
        "\n",
        "    #extra InChI path\n",
        "    extra_InChI_image_dir = './bms-molecular-translation/train_extra'\n",
        "\n",
        "    #hyperparameters\n",
        "    num_epochs = 15\n",
        "    num_folds = 15\n",
        "\n",
        "    #model for original ViT transformer\n",
        "    patch_dim = 64 #dimension of patch for vision part in ViT\n",
        "    encoder_layers = 6\n",
        "    encoder_heads = 3\n",
        "    encoder_one_head_dim = 256\n",
        "    encoder_ff_dim = 512\n",
        "    dropout = 0.1 #use one dropout throughout the ViT model for simplicity\n",
        "    token_dim = 32\n",
        "    decoder_layers = 6\n",
        "    decoder_heads = 3\n",
        "    decoder_one_head_dim = 256\n",
        "    decoder_ff_dim = 512\n",
        "\n",
        "    #model for Resnet18 image encoder\n",
        "\n",
        "    num_channels = 1\n",
        "    pretrained = True\n",
        "\n",
        "    #image params\n",
        "    image_size = 224\n",
        "    patch_size = 16\n",
        "    channels = 1\n",
        "\n",
        "    vocab_length = 175"
      ],
      "id": "cb4e8eb0-68ab-4000-baa0-17dcc75bae1d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "f442ac81-d3cc-4a7b-94ed-fe31bfe37d6a",
        "outputId": "d1b3d3db-0396-4945-9e7e-50eae6e3ef2d"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Read in train and extra dataset. Train shape: (100001, 2), Extra shape: (100001, 1).\n"
          ]
        }
      ],
      "source": [
        "#Load training and extra datas. Note that too large dataset will be computationally costy, even GPU and high-RAM cannot handle it.\n",
        "train = pd.read_csv('./bms-molecular-translation/train_labels.csv')\n",
        "extra = pd.read_csv('./bms-molecular-translation/extra_approved_InChIs.csv')\n",
        "extra = extra.loc[:100000, :]\n",
        "train = train.loc[:100000, :]\n",
        "print('Read in train and extra dataset. Train shape: {}, Extra shape: {}.' \\\n",
        "      .format(train.shape, extra.shape))"
      ],
      "id": "f442ac81-d3cc-4a7b-94ed-fe31bfe37d6a"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 162,
          "referenced_widgets": [
            "279dd157e57f4eb29c5f1ef86d9986d0",
            "a1ded267a2df4c35bd3d3a441f2e098d",
            "22f464229025430cb061b22757ee8fa7",
            "e6188d8108e94c65ab511d3fc78c7817",
            "4b5ac3482ff74e498810a51b6c8db883",
            "bf8915430fee41ef8355e30835f26754",
            "cbf6bd66ae924550945c0efd4efdeeff",
            "db4e8503746a473d9cec26e954b639fd",
            "99382da64dfa48bb8acf59bbd83fb5c4",
            "10eea632aae845e1851798d2a578f679",
            "e52f65136efd4b6b969a9b1718c36e8e",
            "bbf76c24dcf04344978291d39aa83d6a",
            "5207f97ed5494a26a44a0390b79e3150",
            "664b7aa4c30848018e63b23f7df5bea2",
            "9ab8e606e4fe4dc28c655991604d8eba",
            "87a3676e78434e9a83a112a7a35d57b9",
            "2765c90a1c214c91aa6aa9f258751186",
            "ab7484b541084ee7abb1e5defff3bf81",
            "5262f01b7b4440dda0c84d4252cd51f9",
            "b4423f39da644553b89f6bdd390a9811",
            "b7753d0ace004cb183a257f0aece7799",
            "052869ac78eb4f90a82b065a8d959512",
            "65f9e12b2f3a4406880071c2d2f286c4",
            "f42f81aad71c440eb18aba4d5b1f0605",
            "0d2553205e6e4cad9de36e037bfbc2a7",
            "ee7dc8f2a2eb4e00a2cff375554e096c",
            "8adf4a14e5bd44ffb4736718c717fd53",
            "4591a557c0854e039202a136e6570e3f",
            "4f942c7ffd4a44e8937bd8e5216a764f",
            "f0ca9fc0ef6343cdace5066aa00e1146",
            "5572794fb9594b93b04cef0c1ffe6227",
            "cad8971c9ff5436ba1c59acdaea81e09",
            "695dfad19a134d9fb4daf7842609666c",
            "7acdbb3d846d4083a44448219233f5f4",
            "ddc2ada612d140a7a926d73401d509b9",
            "37bac2d8c32f462c96475685a4387410",
            "391eb440cc1b4bdeafa9917af59a6a6a",
            "d15b3a4705424c12816219fe27ef4af1",
            "62d43a8be8634433a64268be873d565a",
            "08ec8ac1bdc74f2dad32da40e7f9b9ca",
            "96c8a4334695410ab5c032f4f5c8b6b4",
            "225e62dc323942bb829ff99971b2865f",
            "5db97629efc04050a09e49b0ba478e20",
            "8e6b5feca4774f77a2a7a855e5eb13e7"
          ]
        },
        "id": "b9fbec95-ae2f-4553-bb47-bbe7b274ccf2",
        "outputId": "0677bb25-68e8-4a35-d8ee-1823778cddde"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Obtain new column of formula and inchi text that will be used for tokenizing.\n"
          ]
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "279dd157e57f4eb29c5f1ef86d9986d0",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/100001 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "bbf76c24dcf04344978291d39aa83d6a",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/100001 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "65f9e12b2f3a4406880071c2d2f286c4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/100001 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "7acdbb3d846d4083a44448219233f5f4",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/100001 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#Tokenize the InChI text. First split the texts to Formula and InChI text two parts. Then split InChI texts to tokens.\n",
        "print('Obtain new column of formula and inchi text that will be used for tokenizing.')\n",
        "train['Formula'] = train['InChI'].progress_apply(lambda x: x.split('/')[1])\n",
        "train['InChI_text'] = train['InChI'].progress_apply(lambda x: split_form(x[9:]))\n",
        "\n",
        "extra['Formula'] = extra['InChI'].progress_apply(lambda x: x.split('/')[1])\n",
        "extra['InChI_text'] = extra['InChI'].progress_apply(lambda x: split_form(x[9:]))"
      ],
      "id": "b9fbec95-ae2f-4553-bb47-bbe7b274ccf2"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "e0b978aa-c0e0-4fca-a23c-3d9daf13b2f6",
        "outputId": "3874e163-e02e-43f6-8dda-74247d02296c"
      },
      "outputs": [
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saved tokenizer\n"
          ]
        }
      ],
      "source": [
        "#Use InChI tokenizer to tokenize InChI text and save the vocab dictionary.\n",
        "tokenizer = Tokenizer()\n",
        "tokenizer.gen_vocab_dict(np.concatenate((train['InChI_text'].values, extra['InChI_text'].values)))\n",
        "torch.save(tokenizer, './bms-molecular-translation/tokenizer.pth')\n",
        "print('Saved tokenizer')\n",
        "#Load the tokenizer.\n",
        "# tokenizer = torch.load('./bms-molecular-translation/tokenizer.pth')"
      ],
      "id": "e0b978aa-c0e0-4fca-a23c-3d9daf13b2f6"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "7U9UyY8nHQYR"
      },
      "outputs": [],
      "source": [
        "# optional, load the tokenizer\n",
        "# tokenizer = torch.load('./bms-molecular-translation/tokenizer.pth')"
      ],
      "id": "7U9UyY8nHQYR"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "e49b8adb-b82b-4115-b87a-4de58bb6dd04"
      },
      "outputs": [],
      "source": [
        "#Create the column 'file_path'based on the column 'id'.\n",
        "train['file_path'] = train['image_id'].apply(train_file_path)"
      ],
      "id": "e49b8adb-b82b-4115-b87a-4de58bb6dd04"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "LWUTNbCdmN0t"
      },
      "outputs": [],
      "source": [
        "#Optional. Save the datasets as .pkl files.\n",
        "train.to_pickle('./bms-molecular-translation/train1.pkl')\n",
        "extra.to_pickle('./bms-molecular-translation/extra1.pkl')"
      ],
      "id": "LWUTNbCdmN0t"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "NZoTI3J_pV6Y"
      },
      "outputs": [],
      "source": [
        "#Optional. Save train/extra ds from the .pkl files.\n",
        "# train = pd.read_pickle('./bms-molecular-translation/train1.pkl')\n",
        "# extra = pd.read_pickle('./bms-molecular-translation/extra1.pkl')"
      ],
      "id": "NZoTI3J_pV6Y"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "41baa185-ab70-462e-8c4a-f19bcfdd5a6d",
        "outputId": "4c35c3a8-67c7-4ac2-ed3c-fb6a6719a8a7"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 100001/100001 [00:00<00:00, 106923.28it/s]\n",
            "100%|██████████| 100001/100001 [00:00<00:00, 109342.83it/s]\n"
          ]
        }
      ],
      "source": [
        "#Create columns that show the counts of individual atoms, such as C, H, O, S, N, etc. Not directly related to the translation goal.\n",
        "#Cannot complete for the limited RAM.\n",
        "train = get_atom_counts(train)\n",
        "extra = get_atom_counts(extra)"
      ],
      "id": "41baa185-ab70-462e-8c4a-f19bcfdd5a6d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "056839ede048437fa9a967491af87b61",
            "8be6e60b1f374dc5a06e8bd54d259e1a",
            "3569ca4f10194535bbb3a51a571a6b97",
            "96a8848677874f52a62832ffbba399a9",
            "bee1f223ee044ebf9b23e7c44d14ea74",
            "71e02a850b1345018b3d6b6c91963600",
            "b384b4c586fc43e09d2de9a657901832",
            "c468246a7da344c79db736cb0f04766c",
            "d31d9201ab0b4308b31fe179632e4c59",
            "4587faae235f4c02824b51e4ec7c3303",
            "d9715f170e5b4f00a6a1f1a3fbe21bea"
          ]
        },
        "id": "549a185f-571f-4cfd-a878-030122dca2be",
        "outputId": "399dd704-c156-47bb-eed7-b9a3cf4fc299"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "056839ede048437fa9a967491af87b61",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/100001 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saved preprocessed train1.pkl\n"
          ]
        }
      ],
      "source": [
        "#Preprocess the training dataset. Create the column of text length.\n",
        "train_text_len = []\n",
        "tk0 = tqdm(train['InChI_text'].values, total = len(train))\n",
        "for text in tk0:\n",
        "    seq = tokenizer.text_to_sequence(text)\n",
        "    length = len(seq) - 2\n",
        "    train_text_len.append(length)\n",
        "train['text_length'] = train_text_len\n",
        "train.to_pickle('./bms-molecular-translation/train1.pkl')\n",
        "print('Saved preprocessed train1.pkl')"
      ],
      "id": "549a185f-571f-4cfd-a878-030122dca2be"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "2qzTvWh_sSrS"
      },
      "outputs": [],
      "source": [
        "#Create the folder for extra training data.\n",
        "!mkdir bms-molecular-translation/train_extra"
      ],
      "id": "2qzTvWh_sSrS"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 66,
          "referenced_widgets": [
            "002c6b943d3e4c2d9ea90103e43b640f",
            "ec9d78130bbe44d6beecf320290a5334",
            "3bb599248e0946aa97ae0f82f5b22563",
            "95601eca7f4842d7a57d9c7f789c0f72",
            "8c8da9cd6039442597b78ddf29d49062",
            "b99f1f1782924394a0f5802384cd7263",
            "7d44c263986c413e9d1d2ca7c6568a4c",
            "39456e84676a4d4da9fc0877509d1fbe",
            "4317a765371d4f549d30bba469090b2d",
            "5159d0ed18ec4ef0b5aa78ad894f8cf7",
            "cf06f265a52d429f805f13804c7d29a6"
          ]
        },
        "id": "6038ba14-824d-48f9-bdbb-671973ec8706",
        "outputId": "d08a87cd-86d0-4ec9-9f3a-b426f77b360e"
      },
      "outputs": [
        {
          "data": {
            "application/vnd.jupyter.widget-view+json": {
              "model_id": "002c6b943d3e4c2d9ea90103e43b640f",
              "version_major": 2,
              "version_minor": 0
            },
            "text/plain": [
              "  0%|          | 0/100001 [00:00<?, ?it/s]"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        },
        {
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Saved preprocessed extra.pkl\n"
          ]
        }
      ],
      "source": [
        "#Generate extra images of mol structures. Save them in the train extra folder.\n",
        "extra.reset_index(inplace = True)\n",
        "extra = extra.rename(columns={'index': 'ids'})\n",
        "extra_dataset_obj = Extra_inchi_save_ds(extra, tokenizer, CFG, transform = None)\n",
        "extra_text_len = []\n",
        "for i in tqdm(range(extra.shape[0])):\n",
        "    seq_length = extra_dataset_obj[i]\n",
        "    extra_text_len.append(seq_length - 2)\n",
        "#Create the text_length and file_path columns\n",
        "extra['text_length'] = extra_text_len\n",
        "extra['file_path'] = './bms-molecular-translation/train_extra/extra_' + extra['ids'].apply(str) + '.png'\n",
        "#Save the extra training dataset.\n",
        "extra.to_pickle('./bms-molecular-translation/extra1.pkl')\n",
        "print('Saved preprocessed extra.pkl')"
      ],
      "id": "6038ba14-824d-48f9-bdbb-671973ec8706"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 466
        },
        "id": "VpFVgZTtnTNf",
        "outputId": "43db950f-ba81-4719-c8ec-d5e621e649dc"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-f41011d7-0c24-49b3-a859-b99c15710d97\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>ids</th>\n",
              "      <th>InChI</th>\n",
              "      <th>Formula</th>\n",
              "      <th>InChI_text</th>\n",
              "      <th>B</th>\n",
              "      <th>Br</th>\n",
              "      <th>C</th>\n",
              "      <th>Cl</th>\n",
              "      <th>F</th>\n",
              "      <th>H</th>\n",
              "      <th>I</th>\n",
              "      <th>N</th>\n",
              "      <th>O</th>\n",
              "      <th>P</th>\n",
              "      <th>S</th>\n",
              "      <th>Si</th>\n",
              "      <th>text_length</th>\n",
              "      <th>file_path</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>InChI=1S/C13H10Cl4O4/c1-20-13(21-2)11(16)5-6-8...</td>\n",
              "      <td>C13H10Cl4O4</td>\n",
              "      <td>C 13 H 10 Cl 4 O 4 /c 1 - 20 - 13 ( 21 - 2 ) 1...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>99</td>\n",
              "      <td>./bms-molecular-translation/train_extra/extra_...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>InChI=1S/C23H32O6Si/c1-20(2)25-15-19(27-20)22(...</td>\n",
              "      <td>C23H32O6Si</td>\n",
              "      <td>C 23 H 32 O 6 Si /c 1 - 20 ( 2 ) 25 - 15 - 19 ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>23</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>32</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>6</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>117</td>\n",
              "      <td>./bms-molecular-translation/train_extra/extra_...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>InChI=1S/C24H16N4O3/c29-21-19-20(14-8-2-1-3-9-...</td>\n",
              "      <td>C24H16N4O3</td>\n",
              "      <td>C 24 H 16 N 4 O 3 /c 29 - 21 - 19 - 20 ( 14 - ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>16</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>3</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>106</td>\n",
              "      <td>./bms-molecular-translation/train_extra/extra_...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>InChI=1S/C20H26N2O5/c1-9-14-11-7-13-19(8-21-3)...</td>\n",
              "      <td>C20H26N2O5</td>\n",
              "      <td>C 20 H 26 N 2 O 5 /c 1 - 9 - 14 - 11 - 7 - 13 ...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>20</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>26</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>5</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>125</td>\n",
              "      <td>./bms-molecular-translation/train_extra/extra_...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>InChI=1S/C14H12BrN/c15-8-3-1-7(2-4-8)12-9-5-10...</td>\n",
              "      <td>C14H12BrN</td>\n",
              "      <td>C 14 H 12 Br N /c 15 - 8 - 3 - 1 - 7 ( 2 - 4 -...</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>14</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>12</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>81</td>\n",
              "      <td>./bms-molecular-translation/train_extra/extra_...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-f41011d7-0c24-49b3-a859-b99c15710d97')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-f41011d7-0c24-49b3-a859-b99c15710d97 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-f41011d7-0c24-49b3-a859-b99c15710d97');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-e8d60fbc-ee9a-4e7c-8317-8e06c7f4e300\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-e8d60fbc-ee9a-4e7c-8317-8e06c7f4e300')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-e8d60fbc-ee9a-4e7c-8317-8e06c7f4e300 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "   ids                                              InChI      Formula  \\\n",
              "0    0  InChI=1S/C13H10Cl4O4/c1-20-13(21-2)11(16)5-6-8...  C13H10Cl4O4   \n",
              "1    1  InChI=1S/C23H32O6Si/c1-20(2)25-15-19(27-20)22(...   C23H32O6Si   \n",
              "2    2  InChI=1S/C24H16N4O3/c29-21-19-20(14-8-2-1-3-9-...   C24H16N4O3   \n",
              "3    3  InChI=1S/C20H26N2O5/c1-9-14-11-7-13-19(8-21-3)...   C20H26N2O5   \n",
              "4    4  InChI=1S/C14H12BrN/c15-8-3-1-7(2-4-8)12-9-5-10...    C14H12BrN   \n",
              "\n",
              "                                          InChI_text  B  Br   C  Cl  F   H  I  \\\n",
              "0  C 13 H 10 Cl 4 O 4 /c 1 - 20 - 13 ( 21 - 2 ) 1...  0   0  13   4  0  10  0   \n",
              "1  C 23 H 32 O 6 Si /c 1 - 20 ( 2 ) 25 - 15 - 19 ...  0   0  23   0  0  32  0   \n",
              "2  C 24 H 16 N 4 O 3 /c 29 - 21 - 19 - 20 ( 14 - ...  0   0  24   0  0  16  0   \n",
              "3  C 20 H 26 N 2 O 5 /c 1 - 9 - 14 - 11 - 7 - 13 ...  0   0  20   0  0  26  0   \n",
              "4  C 14 H 12 Br N /c 15 - 8 - 3 - 1 - 7 ( 2 - 4 -...  0   1  14   0  0  12  0   \n",
              "\n",
              "   N  O  P  S  Si  text_length  \\\n",
              "0  0  4  0  0   0           99   \n",
              "1  0  6  0  0   1          117   \n",
              "2  4  3  0  0   0          106   \n",
              "3  2  5  0  0   0          125   \n",
              "4  1  0  0  0   0           81   \n",
              "\n",
              "                                           file_path  \n",
              "0  ./bms-molecular-translation/train_extra/extra_...  \n",
              "1  ./bms-molecular-translation/train_extra/extra_...  \n",
              "2  ./bms-molecular-translation/train_extra/extra_...  \n",
              "3  ./bms-molecular-translation/train_extra/extra_...  \n",
              "4  ./bms-molecular-translation/train_extra/extra_...  "
            ]
          },
          "execution_count": 23,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Optional, check the columns' names of extra and train dataset.\n",
        "# extra.head()"
      ],
      "id": "VpFVgZTtnTNf"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 486
        },
        "id": "dZyGISZ-mvRP",
        "outputId": "93e5af3c-205c-4817-f901-9941eefb7a0f"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-ea453636-2a3a-475f-aa97-72f74d868ebf\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>image_id</th>\n",
              "      <th>InChI</th>\n",
              "      <th>Formula</th>\n",
              "      <th>InChI_text</th>\n",
              "      <th>file_path</th>\n",
              "      <th>B</th>\n",
              "      <th>Br</th>\n",
              "      <th>C</th>\n",
              "      <th>Cl</th>\n",
              "      <th>F</th>\n",
              "      <th>H</th>\n",
              "      <th>I</th>\n",
              "      <th>N</th>\n",
              "      <th>O</th>\n",
              "      <th>P</th>\n",
              "      <th>S</th>\n",
              "      <th>Si</th>\n",
              "      <th>text_length</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>000011a64c74</td>\n",
              "      <td>InChI=1S/C13H20OS/c1-9(2)8-15-13-6-5-10(3)7-12...</td>\n",
              "      <td>C13H20OS</td>\n",
              "      <td>C 13 H 20 O S /c 1 - 9 ( 2 ) 8 - 15 - 13 - 6 -...</td>\n",
              "      <td>./bms-molecular-translation/train/0/0/0/000011...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>13</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>20</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>59</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>000019cc0cd2</td>\n",
              "      <td>InChI=1S/C21H30O4/c1-12(22)25-14-6-8-20(2)13(1...</td>\n",
              "      <td>C21H30O4</td>\n",
              "      <td>C 21 H 30 O 4 /c 1 - 12 ( 22 ) 25 - 14 - 6 - 8...</td>\n",
              "      <td>./bms-molecular-translation/train/0/0/0/000019...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>21</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>30</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>0000252b6d2b</td>\n",
              "      <td>InChI=1S/C24H23N5O4/c1-14-13-15(7-8-17(14)28-1...</td>\n",
              "      <td>C24H23N5O4</td>\n",
              "      <td>C 24 H 23 N 5 O 4 /c 1 - 14 - 13 - 15 ( 7 - 8 ...</td>\n",
              "      <td>./bms-molecular-translation/train/0/0/0/000025...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>23</td>\n",
              "      <td>0</td>\n",
              "      <td>5</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>112</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>000026b49b7e</td>\n",
              "      <td>InChI=1S/C17H24N2O4S/c1-12(20)18-13(14-7-6-10-...</td>\n",
              "      <td>C17H24N2O4S</td>\n",
              "      <td>C 17 H 24 N 2 O 4 S /c 1 - 12 ( 20 ) 18 - 13 (...</td>\n",
              "      <td>./bms-molecular-translation/train/0/0/0/000026...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>17</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>24</td>\n",
              "      <td>0</td>\n",
              "      <td>2</td>\n",
              "      <td>4</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>108</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>000026fc6c36</td>\n",
              "      <td>InChI=1S/C10H19N3O2S/c1-15-10(14)12-8-4-6-13(7...</td>\n",
              "      <td>C10H19N3O2S</td>\n",
              "      <td>C 10 H 19 N 3 O 2 S /c 1 - 15 - 10 ( 14 ) 12 -...</td>\n",
              "      <td>./bms-molecular-translation/train/0/0/0/000026...</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>10</td>\n",
              "      <td>0</td>\n",
              "      <td>0</td>\n",
              "      <td>19</td>\n",
              "      <td>0</td>\n",
              "      <td>3</td>\n",
              "      <td>2</td>\n",
              "      <td>0</td>\n",
              "      <td>1</td>\n",
              "      <td>0</td>\n",
              "      <td>72</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-ea453636-2a3a-475f-aa97-72f74d868ebf')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-ea453636-2a3a-475f-aa97-72f74d868ebf button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-ea453636-2a3a-475f-aa97-72f74d868ebf');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-309b6697-18c7-4188-ae01-8c583e727146\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-309b6697-18c7-4188-ae01-8c583e727146')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-309b6697-18c7-4188-ae01-8c583e727146 button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "       image_id                                              InChI  \\\n",
              "0  000011a64c74  InChI=1S/C13H20OS/c1-9(2)8-15-13-6-5-10(3)7-12...   \n",
              "1  000019cc0cd2  InChI=1S/C21H30O4/c1-12(22)25-14-6-8-20(2)13(1...   \n",
              "2  0000252b6d2b  InChI=1S/C24H23N5O4/c1-14-13-15(7-8-17(14)28-1...   \n",
              "3  000026b49b7e  InChI=1S/C17H24N2O4S/c1-12(20)18-13(14-7-6-10-...   \n",
              "4  000026fc6c36  InChI=1S/C10H19N3O2S/c1-15-10(14)12-8-4-6-13(7...   \n",
              "\n",
              "       Formula                                         InChI_text  \\\n",
              "0     C13H20OS  C 13 H 20 O S /c 1 - 9 ( 2 ) 8 - 15 - 13 - 6 -...   \n",
              "1     C21H30O4  C 21 H 30 O 4 /c 1 - 12 ( 22 ) 25 - 14 - 6 - 8...   \n",
              "2   C24H23N5O4  C 24 H 23 N 5 O 4 /c 1 - 14 - 13 - 15 ( 7 - 8 ...   \n",
              "3  C17H24N2O4S  C 17 H 24 N 2 O 4 S /c 1 - 12 ( 20 ) 18 - 13 (...   \n",
              "4  C10H19N3O2S  C 10 H 19 N 3 O 2 S /c 1 - 15 - 10 ( 14 ) 12 -...   \n",
              "\n",
              "                                           file_path  B  Br   C  Cl  F   H  I  \\\n",
              "0  ./bms-molecular-translation/train/0/0/0/000011...  0   0  13   0  0  20  0   \n",
              "1  ./bms-molecular-translation/train/0/0/0/000019...  0   0  21   0  0  30  0   \n",
              "2  ./bms-molecular-translation/train/0/0/0/000025...  0   0  24   0  0  23  0   \n",
              "3  ./bms-molecular-translation/train/0/0/0/000026...  0   0  17   0  0  24  0   \n",
              "4  ./bms-molecular-translation/train/0/0/0/000026...  0   0  10   0  0  19  0   \n",
              "\n",
              "   N  O  P  S  Si  text_length  \n",
              "0  0  1  0  1   0           59  \n",
              "1  0  4  0  0   0          108  \n",
              "2  5  4  0  0   0          112  \n",
              "3  2  4  0  1   0          108  \n",
              "4  3  2  0  1   0           72  "
            ]
          },
          "execution_count": 24,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Optional, check the columns' names of extra and train dataset.\n",
        "# train.head()"
      ],
      "id": "dZyGISZ-mvRP"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 452
        },
        "id": "7e9da4a3-8439-429b-9fc2-ca195ed950a9",
        "outputId": "8dc01eef-2435-49e0-9801-881caf48d5ad"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "<matplotlib.image.AxesImage at 0x7d11d6d5dff0>"
            ]
          },
          "execution_count": 25,
          "metadata": {},
          "output_type": "execute_result"
        },
        {
          "data": {
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAakAAAGiCAYAAABd6zmYAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAACE2UlEQVR4nO3dd3yb1dn4/4/kIe+9Z+x4xvFMYsfZZAfIIGGEURJKQ6EJD5QWeCgUSksfaNNSWgrdX6Cl7CZAQjbZibOcbSeOHe8h2/GSty3p/v3BTyomgwzbkuzr/XrpBZFuS+eW5fvSOec611EpiqIghBBCWCG1pRsghBBCXI4EKSGEEFZLgpQQQgirJUFKCCGE1ZIgJYQQwmpJkBJCCGG1JEgJIYSwWhKkhBBCWC0JUkIIIayWBCkhhBBWy2JB6o033mDEiBE4OTmRlZXFoUOHLNUUIYQQVsoiQerDDz/kiSee4IUXXuDo0aOkpqYyZ84c6urqLNEcIYQQVkpliQKzWVlZjBs3jj/+8Y8AGI1GwsPDefTRR/nf//3fwW6OEEIIK2U/2C/Y09NDbm4uzzzzjPk+tVrNzJkzycnJueTPdHd3093dbf630WiksbERX19fVCrVgLdZCCFE/1IUhdbWVkJCQlCrLz+oN+hB6sKFCxgMBgIDA/vcHxgYyNmzZy/5My+//DIvvvjiYDRPCCHEIKqoqCAsLOyyjw96kLoezzzzDE888YT53y0tLURERFBRUYGHh4cFWyaEEOJ66HQ6wsPDcXd3v+Jxgx6k/Pz8sLOzo7a2ts/9tbW1BAUFXfJnNBoNGo3movs9PDwkSAkhhA37timbQc/uc3R0ZMyYMXz55Zfm+4xGI19++SXZ2dmD3RwhhBBWzCLDfU888QTLli1j7NixZGZm8tprr9He3s4DDzxgieYIIYSwUhYJUnfddRf19fU8//zzaLVa0tLS2LRp00XJFEIIIYY3i6yTulE6nQ5PT09aWlpkTkoIIWzQ1V7HpXafEEIIqyVBSgghhNWSICWEEMJqSZASQghhtSRICSGEsFoSpIQQQlgtCVJCCCGslgQpIYQQVkuClBBCCKslQUoIIYTVkiAlhBDCakmQEkIIYbUkSAkhhLBaEqSEEEJYLQlSQgghrJYEKSGEEFZLgpQQQgirJUFKCCGE1ZIgJYQQwmpJkBJCCGG1JEgJIYSwWhKkhBBCWC0JUkIIIayWBCkhhBBWS4KUEEIIqyVBSgghhNWSICWEEMJqSZASQghhtSRICSGEsFoSpIQQQlgtCVJCCCGslgQpIYQQVkuClBBCCKslQUoIIYTVkiAlhBDCakmQEkIIYbUkSAkhhLBaEqSEEEJYLQlSQgghrJYEKSGEEFZLgpQQQgirJUFKCCGE1ZIgJYQQwmpJkBJCCGG1JEgJIYSwWhKkhBBCWC0JUkIIIayWBCkhhBBWS4KUEEIIqyVBSgghhNWSICWEEMJq9XuQ+tnPfoZKpepzS0hIMD/e1dXFypUr8fX1xc3NjSVLllBbW9vfzRBCCDEEDEhPKikpiZqaGvNt79695sd++MMfsm7dOj7++GN27dpFdXU1ixcvHohmCCGEsHH2A/Kk9vYEBQVddH9LSwv/+Mc/eO+995g+fToAb731FomJiRw4cIDx48cPRHOEEELYqAHpSRUWFhISEkJ0dDT33nsv5eXlAOTm5tLb28vMmTPNxyYkJBAREUFOTs5ln6+7uxudTtfnJoQQYujr9yCVlZXF22+/zaZNm/jTn/5ESUkJkydPprW1Fa1Wi6OjI15eXn1+JjAwEK1We9nnfPnll/H09DTfwsPD+7vZQgghrFC/D/fNmzfP/P8pKSlkZWURGRnJRx99hLOz83U95zPPPMMTTzxh/rdOp5NAJYQQw8CAp6B7eXkRFxdHUVERQUFB9PT00Nzc3OeY2traS85hmWg0Gjw8PPrchBBCDH0DHqTa2to4f/48wcHBjBkzBgcHB7788kvz4wUFBZSXl5OdnT3QTRFCCGFj+n2478c//jHz588nMjKS6upqXnjhBezs7Lj77rvx9PTkwQcf5IknnsDHxwcPDw8effRRsrOzJbNPCCHERfo9SFVWVnL33XfT0NCAv78/kyZN4sCBA/j7+wPwu9/9DrVazZIlS+ju7mbOnDm8+eab/d0MIYQQQ4BKURTF0o24VjqdDk9PT1paWmR+SgghbNDVXseldp8QQtiQuro6jhw5wqeffooN9jGu2YBUnBBCCNH/jEYjubm55OXl4evra+nmDArpSQkhhI2oq6sjJyeHkydPEhYWZunmDArpSQkhhJUzDet99NFHVFZWEhcXx6xZsyzcqsEhPSkhhLByiqJQUFDAv/71L0aPHs2KFSss3aRBI0FKCCGsmKIodHV18dOf/pS5c+cyadKkYTMfBRKkhBDCqrW0tLBr1y5qa2uZOnUqI0eOxN5++MzUDJ8zFUIMK19Pz1apVBZsyfXr6emhurqadevWkZycTHJyMj4+PpZu1qCSICWEGJK6urowGo3Y2dmh0WhsLlApikJDQwMnTpxg8+bNbN26FR8fH5s7jxslQUoIMSS9//77HD58mBEjRrBq1SpcXV0t3aRroigKX3zxBV988QVPP/00I0aMGHYBCmROSggxRC1ZsoT09HROnz7NAw88QEVFBQaDwdLNumrr1q3jxIkTBAYGsnTpUlQqlQQpIYQYKtzd3ZkxYwYLFy4kJCSE559/nuPHj9PW1mbppl2RoijU19fzxRdf4OjoyKJFi3B3d7d0syxGhvuEEEOSWq1m5MiRuLq64uDgwDvvvMMXX3xBa2srCQkJV9xo1VIURUFRFHbt2kVzczOpqamMGzduWPagTCRICSGGtKCgIObNm0dQUBCPP/441dXVzJkzhxkzZph7KNYSBIxGI62trfzlL39hxowZZGdnD6s1UZciw31CiCHP0dGRzMxM3n//fTo7O3nzzTf5zW9+Q09PD4BVVBNXFIWWlhb+9a9/0dXVxS233EJaWpqlm2VxEqSEEMNGeHg4v/3tb3nwwQc5ffo0S5Ysobq6Gr1eb+mm0dbWRl5eHr/97W955ZVXiIqKspoeniVJkBJCDAsqlQo7Ozu8vb2ZNm0ajz76KHFxcfz4xz9m7969NDY2WqxtiqJw4sQJ1q5dyy233MKoUaNwdnaWIIUEKSHEMGNnZ0dgYCDjxo1jzpw52NnZsXnzZnbt2kVVVZVF2lReXs7JkycpLy9n0aJFeHh4YGdnZ5G2WBtJnBBCDDsqlQo3Nzdmz56NSqXizTffpLGxkd7eXm666Sb8/PzMxw0kRVEwGAzs37+fgoICAgMDmT59Omq19B9M5J0QQgxbKpWK2bNns3r1anx9ffnNb37D73//e3p7e4HBSaioq6vjk08+obu7m8cffxw7OzsZ5vsaCVJCiGFv5MiR/O///i+//vWv2b9/P8uXL6ekpGRQEiqeffZZoqKimD9/PiNHjhzw17M1EqSEEMOeWq3Gzc2NjIwMVq9ejUaj4bnnnuPzzz8fsISKrq4uNmzYQHFxMZMmTSI7O1uG+S5B3hEhhOCrhAp3d3eSk5NZvHgxAQEB7Nu3j48//piysrJ+Hfrr7e3lwoULvPfee0yePJnExES8vLz67fmHEglSQgjx/1OpVDg6OjJ//nxuvfVWent72bVrF9u3b0er1fZbgdqWlhZOnjzJqVOnWLRoEaGhodKLugzJ7hNCiEuYOXMmI0aMYMOGDaxevZrm5mYeeOAB3N3dUavV153cYDAYyMvL4+233+aOO+4gKSkJZ2fnfm790CFBSgghLmPkyJF897vfZeLEiSxbtoxTp07xyCOPkJ6eft1buB8+fJitW7dSW1vL22+/jZOTUz+3emiR/qUQQlyGSqXCxcWFUaNG8cEHH2AwGPjtb3/Lm2++SVNT0zU/X0tLC2vWrKG0tJRnn31WAtRVkCAlhBBXoFarcXJyIi4ujvvvv5/4+HgKCgr44x//SHV1NUaj8aqfa+3atbS1tTFq1CgyMjJuaNhwuJAgJYQQ38KUUDF9+nRmzpxJUFAQJ0+eZMOGDVRXV5urqV+O0WhEq9Wybds2/P39yc7ONle1EFcmc1JCCHGVVCoVkydPJiIigm3btvHKK6+gUqmYOXMmwcHBODg4XNQzUhSF3t5etm/fTlVVFUuWLGHixIkWOgPbo1KsYSOVa6TT6fD09KSlpQUPDw9LN0cIMcwoikJPTw95eXncc889TJgwgdtvv5158+ZdFKR6e3upqakhOzubP/7xj0yYMIGAgIBhP8x3tddxGe4TQohrZBr+S05OZs2aNahUKv72t7/xi1/8gs7Ozj4Lf8+fP88rr7zCggULzMN8wz1AXQsJUkIIcR1UKhX29vZER0fz0EMPMWXKFEpKSnjuueeora1Fr9dTWVlJbm4up06dYvny5Xh7e8sWHNdI5qSEEOI6qVQqnJycGDNmDBqNBpVKxcGDB/nkk0+YPn06eXl5HDt2jMTERFJTU3F0dLR0k22OBCkhhLhB9vb2pKWlERERgZ+fH7/5zW/o6uri2LFjtLa28txzz5mDmLg2EqSEEKKfeHt7c8899+Dr68szzzxDV1cX06dPJyEhAfjv/lQSrK6ezEkJIUQ/6unp4aOPPsLHx4fY2FjKy8vJysri9ddf58KFC/1WpHa4kJ6UEEL0A9OC3b///e80NDTwwx/+kIiICHp6eigoKGDfvn08/PDDjBkzhsmTJzN+/HgcHBws3WyrJ0FKCCFukNFo5Ny5c+zdu5dTp05x8803k5mZiZ+fHwaDgdDQUBwcHMjNzaWuro4tW7Zw5swZMjMzSUpKwt7eXoYAL0OClBBCXCdFUVAUhaqqKvbs2cPWrVsJDg7m/vvvx8nJCbVajZ2dHSEhIdx1111kZWWxbds2cnJyOHToEE1NTSiKQlBQEJ6enjg7O0uw+gapOCGEENfBdOns7OzkpZdeIj8/nxEjRvCrX/0KR0fHKwabCxcucOzYMVavXk1tbS133nkns2bNIi0tzdyrGurB6mqv4xKkhBDiOphKIz3wwAP09PRw6623cvfdd5vXQl0pyJh6YEajkV27dvHWW29RUFCAl5cXDz/8MIsWLRryi34lSAkhxAAxGAzU1dXxy1/+krq6OlasWEFGRgY+Pj7X1ANSFIX29nYaGhooKyvjxIkTrFu3jsDAQO68804yMjIIDQ0dwDOxnKu9jsuclBBCXAO9Xs/Zs2fZsmULWq2WpUuXkpaWds0BCr7qbbm5ueHm5oaPjw/+/v64u7tTUFDAli1bOHbsGAkJCUyYMIHQ0NAhPwR4KRKkhBDiKhkMBs6fP8+ePXvYu3cv6enp3Hzzzf2S8ODu7k5CQgJxcXHs37+fNWvWUFBQgFarpaOjg/T0dMLDw3F3dx9Wqesy3CeEEN/CdJlsbGzk1VdfJT8/n7i4OH75y19iZ2c3ID0cvV7P8ePH2bRpE++99x5xcXGsWLGC5ORkAgICcHR0RK223XoMMiclhBD9RFEUOjo6uP/++3F0dGTWrFksW7bMHCQGIkiZLs2KotDZ2cm///1vXn/9dby9vZk2bRrLly8nKirKZocAJUgJIUQ/0Ov1NDQ08Oijj+Li4sL999/PuHHjcHNzG5QAYcoEbG9vp76+ngMHDrBnzx4OHTrE7NmzWbFiBSEhITg5OQ14W/qTJE4IIcQN6unp4cyZM3z44YfY2dlx3333kZKSMmgBCjCvmXJ3d8fZ2RlHR0fCw8NJT0/nzJkzvPTSS6SmpjJ+/HjGjRt30RCgwWCgtbWVQ4cOmWsHOjs7ExISwtixYy+55b01kSAlhBCX0NvbS35+Prt37yYvL49bbrmF7OxsXFxcLHZRt7e3JywsjKCgIEaNGsWmTZv48ssvqaioIDIy8qLj29vbzdUwTp8+TW9vL4qiYGdnh4uLC7W1tUyePBlPT0+rTca45lm33bt3M3/+fEJCQlCpVHz66ad9HlcUheeff57g4GCcnZ2ZOXMmhYWFfY5pbGzk3nvvxcPDAy8vLx588EHa2tpu6ESEEKI/mBbZVldXs3btWrZv387YsWNZsWKFRQPU19nb2+Pr68u9997Lr371K2bMmIGfn1+fXpTRaKSsrIzPP/+cn/zkJ5w+fRoHBwdcXV1pa2tjy5YtPPnkkxw5cgSdTmfBs/kWyjXasGGD8uyzzypr1qxRAGXt2rV9Hn/llVcUT09P5dNPP1VOnDihLFiwQImKilI6OzvNx8ydO1dJTU1VDhw4oOzZs0eJiYlR7r777qtuQ0tLiwIoLS0t19p8IYS4LKPRqBiNRqWxsVFZtGiRcvfddyv/+Mc/LN2s66LT6ZSXX35ZiYqKUh5++GGlp6dHMRqNiqIoSm9vr6LVapW0tDQlOztb2bhxo/mxwXK11/FrDlJ9fvgbQcpoNCpBQUHK6tWrzfc1NzcrGo1Gef/99xVFUZT8/HwFUA4fPmw+ZuPGjYpKpVKqqqqu6nUlSAkhBkJPT49SWVmpTJ48WXn44YeV7du3K11dXZZu1nXZsGGDsnTpUiU9PV0pLS01B2BF+epa3d3drXz44YdKUFCQ8qtf/UqprKwc1PZd7XW8X5PsS0pK0Gq1zJw503yfp6cnWVlZ5OTkAJCTk4OXlxdjx441HzNz5kzUajUHDx685PN2d3ej0+n63IQQoj91dXWRl5fHCy+8QGxsLPfffz9paWloNBpLN+26lJWV0d7eTmBgIEFBQcB/U+VVKhX29vZkZGSgVquprq6murraks29rH4NUlqtFoDAwMA+9wcGBpof02q1BAQE9Hnc3t4eHx8f8zHf9PLLL+Pp6Wm+hYeH92ezhRCCCxcuUFxcTHNzM/Pnzyc5ORlvb29LN+u6tbS0YDAY8PT0RKPRXDSXplKpCAoKwt7entbWVlpbWy3U0iuzieXKzzzzDC0tLeZbRUWFpZskhBhiWlpa6O7uZvbs2SxYsAA3NzdLN+mGGAwGVCqVuSr7pWg0GtRqNUaj0Wq3te/XFHRTl7K2tpbg4GDz/bW1taSlpZmPqaur6/Nzer2exsZG889/k0ajsdkutxDCNiQlJZGUlGTpZvQbV1dXFEWhubnZXL3i670pRVFoaGhAr9fj7OyMi4uLpZp6Rf3ak4qKiiIoKIgvv/zSfJ9Op+PgwYNkZ2cDkJ2dTXNzM7m5ueZjtm/fjtFoJCsrqz+bI4QQw1ZcXBzu7u6Ul5dTWVl50eMGg4G9e/fS3d1NVFQU0dHRFmjlt7vmnlRbWxtFRUXmf5eUlHD8+HF8fHyIiIjg8ccf56WXXiI2NpaoqCh++tOfEhISwqJFiwBITExk7ty5rFixgj//+c/09vayatUqli5dSkhISL+dmBBieGtsbOTs2bNUVlbS1taGi4sLqampRERE4OrqaunmDbgxY8aQm5vL8ePHeeONN/j5z39uri5hKvX017/+lfj4eBISEvDz87N0ky/pmoPUkSNHuOmmm8z/fuKJJwBYtmwZb7/9Nk899RTt7e089NBDNDc3M2nSJDZt2tSnrtS///1vVq1axYwZM1Cr1SxZsoQ//OEP/XA6QojhzmAwcOLECT7//HPq6urQ6/U4OjrS3d3Nrl27yMjIYNy4caSkpNh0FfFv4+3tTVZWFhUVFezcuZM//OEPhIWFYWdnR2trK+fPn6e2tpb777+f+Ph4q604IQVmhRBDhl6vp6Wlhd/+9rds3LiRoKAgIiMj8fPzo6GhgYMHD+Lj48OcOXO49957h/zoTX19PUePHuV3v/sdXV1dREREoFaraW1tpaamhvj4eJ599lnCw8MHfd5fqqALIYadlpYWjhw5wrx583juuee48847SUhIAL5KFNi3bx/PPvssiqJwzz338PDDD1u4xQPPVGB2w4YNlJaWotfr8fDwIDExkRkzZgzYfljfRoKUEGLYyc/P55VXXuHQoUNs3bqV0NDQPkN6iqLwxhtvsH79erq6uti5c6flGjtIvn6J//r/fz0wWXOQGroDskKIYae1tZUzZ84wZswYnJycLrmANT4+noCAAGpra9Fqtdjg9/RrYtrqQ6VSoVarzbev32/NJEgJIYYM05xUYGDgZYexfHx8cHd3p6enh9bW1iEfpJT/f9NE083WSJASQgwZyv+/zcaV5llMPQnAaqss9Cej0UhbWxu7d++mvb3d0s25ZhKkhBBDhr29Pa6urjQ0NFw2AOl0Otrb27Gzs8Pd3d3qh7tuVG9vLxUVFTz22GNWW0T2SiRICSGGDDc3NxITEzl27Bg9PT2XPKawsJD6+noCAgKGfAo6fLWLxLlz5wgPD7fatVBXIkFKCDFkBAUFcfvtt5OXl8eGDRv6FKNWFIXKykq++OILWlpaWLBggQVbOni6urooLCwkNjbWJmug9muBWSGEsCQPDw8yMzNZvHgx77zzDsePHychIQFfX18aGxvZs2cPFy5cYPz48cyfP3/ID/UB9PT0UFZWRmxs7BUrolsr6UkJIYYMR0dHgoODufvuu4mMjKShoYETJ05w8OBBTpw4QUdHB5MnT2b27NnExMRYurmDoqenh8rKSpsd7pOelBBiSHFwcGDhwoXEx8dz5MgRioqK0Ol05kLX48aNw8/Pb1j0ohRFMfekIiIibLInJUFKCDEkJSQkmEsiDWddXV2UlZURHR1tk0FKhvuEEGKIamlpoa6uDl9fX9zc3Gyy6rvttVgIIfqBaeGvLVZhuFrNzc3U19czYsQIcykkWyNBSggx7Oh0OgoKCvj0008xGo2Wbs6A0el0NDY2EhERYemmXDcJUkKIYaejo4OioiL+9re/odPphmx5JJ1OR0NDAxERETbZiwIJUuI6Xalgpel+01DKN7cK+PrjpputFr8Utsnf35+YmBiKioo4efIkHR0dlm7SgGhubqauro6RI0dKkBLDS0VFBWfPnqW8vPyixwwGA83NzeTk5NDW1nbRcIqiKJw9e5YdO3awbt06tm/fTkNDw5AedhHWRa1WExAQwP/8z//w2muvUVNTY+kmDYiGhgbzDry2SlLQxXVZu3YtZ86cISYmhh//+Md9Hmtvb+fkyZN873vf45NPPiE2NhYXFxe6urooLi7mpz/9KcXFxWg0GjQaDe3t7ej1eu644w5uu+02EhISbDILSdgOlUqFh4cHS5Ys4e9//zt5eXl4eXkREBBg6ab1G6PRSHNzMw0NDURFRdlsT0qClLgujY2NaLVavL29L3rMYDDQ0dHB+fPn6e7uRlEUcyXmP/7xjzQ2NnLzzTcTEhKCk5MTHR0d5ObmsmXLFpycnDAYDCQnJ1vgrMRwYmdnh7+/P+PHj+fo0aP4+fkNqSCl0+no6urC0dHRpncwlyAlBkVzczOnT5/mP//5Dw888AArVqwgNDQUBwcH9Ho9+/bt46mnnmL37t14enqSlJQkvSkxoFQqFXZ2dsyfP593332X8PBwMjIycHV1tXTT+kVDQwN6vR5vb2+bXMRrIlcBG2HNO2tezc6f58+fZ9euXRgMBh577DHCwsLMdcTs7e2ZOnUqixYtoqamhn379tHb2zuYpyCGKZVKxZw5czAYDBQVFVFYWGjpJvWb6upqDAYDYWFhlm7KDZEgZSO6u7tpa2ujs7PTagKVKUNPr9dfdPtmSm9VVRWFhYWMHTsWf39/7OzsLnq+6dOn4+3tTXV19ZC6WAjrZm9vz9KlS+nt7eXf//63pZvTb8rLyzEYDERFRVm6KTdEhvtsxOHDh80buf3oRz+ydHMAKC0tZefOnXzwwQd97jcajXR3d/e5r7Ozk/b2dvOiwktN4gYEBODs7Gye7BVioJk+h7NmzaKoqIi9e/dy8uRJkpOTbTbRwKSyshKDwWDTC3lBgpTNaGpq4sKFC1Y1Aert7U1AQADz58/vc39HRwdlZWX85S9/Md+nVqtRq9WX3S0VvtpSwGg0olKpsLeXj6YYPG5ubiQlJVFTU8Mnn3zC6NGjbT5I1dTU4OLiYvO7D8uVwEZ0dHTQ3d2Nl5eXpZti5u7uTkxMDPfdd1+f+5ubm8nNze0TpJycnHBxcaGhocE8d/XNi0BDQwNdXV1oNBqrOk8x9KnVahISEqitreW9996joaEBHx+fSw5LWzvTdEB9fT3h4eEEBgZauEU3RuakbERbWxvd3d34+PhYuilmdnZ2ODk54ePj0+fm5eWFm5tbn2O9vb0JDg6mrKzsonk1U9AqKiqio6MDT09Pm//2J2xPVFQUqampNDc3c+zYsYuGrG1Jb28vDQ0NqFQqCVJicLS2ttLZ2Wmz6zji4uKYOHEiJSUlrF+/Hp1O1ycbsLu7m3fffRd7e3syMjIuuf5KiIFkZ2dHWFgYDz74IL///e9pamqymiSla1VZWYlGo8HT0xNnZ2dLN+eGyHCfjdDpdHR2duLv72/pplyX4OBgpk2bxve+9z1++MMfsmfPHpKTk/Hx8aGhoYFPP/2UoqIiHnvsMRYvXmzp5ophyt/fnzvvvJPXXnuNY8eO4ejoaJN/c+fOncPPz88m2/5NEqRsRFtbG11dXVYz3BceHk53dzeRkZEXPWZvb4+3tzfZ2dnmjdbUajVBQUGsWrWKoKAgysvL2b17NwaDAZVKRVRUFA888ACZmZk2PzwhbJdarcbd3Z3FixezdetWvLy8bPJCX1paire3t9VcL26EBCkboNfr6e7uxmg0XjTXYynp6elERERcclhOo9EQGhrK8uXL8fPzM2fqOTk5kZiYyKJFi8jLy+PChQt0d3fj6OhIVFQU2dnZeHl5mRf5CjHYTJmlN998M6tXr+b8+fMkJSXZxPBzd3c3ra2tVFZWcuTIEUJCQoZEApIEKRvQ2dmJwWDA3t7easaXx4wZc9nHNBoN4eHhrFix4qLH1Go1qamppKamDmTzhLhuarWa7OxsHBwcKCkpoaioiHHjxlm6WX0YDAZ6e3vNWb89PT00NTVRWlrKwYMH2b17N8uXL7eJ4PptJEjZgMbGRuzt7a1qjdRg+vrkta2vXRHWT6VS4eDgwF133cWuXbv44osvGDt2rMU+e6bP/9f/29raai41VlBQQGFhIaWlpVRVVdHT04OLiwsBAQH4+flZpM39SYKUDaivr8fBwcFqelGDrbGxkd7eXhwdHYfEGLuwDQsXLuTEiROcPn2aI0eOWCxQdXZ2UlRUxLZt2ygsLKSsrIyysjLKy8vp6uq6ZO3M7u5uQkNDJUiJwdHU1ISdnd2w6kkpioLBYGDjxo1otVoiIyPJzs62dLPEMKLRaMjOzsZgMPDvf/+bsWPHDujrKYpCZ2cnhYWFnDp1itLSUmpqaqiurqa0tNS82L27u9s8xHe5FHlnZ2dcXV1tuvq5iQQpG9Dc3Iy9vT3u7u6WbsqgaGtrMxektbe3Jzo6mpCQEDQajaWbJoYRlUpFcnIydXV1vPfee5SXlxMWFtYvVShMe6xVVlZSXl5OY2MjTU1N1NfXc+7cOSoqKrhw4QLNzc20tLTQ0tJyTTtXm7bnGArb3UiQsgHNzc3Y2dlZTWbfQOnt7aWxsZG6ujoqKiooKiri5ptvNl8YOjs76enpwdXVVeamxKAIDw8nNjYWtVpNTk4OCxcuvOZhd9OogE6no729na6uLjo7O9HpdBw9epRTp05RWVmJVqs1325UcHDwkOhFgQQpmzBchvuamprYuHEjjY2NZGRk8NhjjwFfZTJVVlZSUlKCp6cnaWlpEqS+xdcn2U3fwFUqFWq12iLvnak9RqOxz81Uw1Gj0Vjlt347OztGjBjB4sWLeeutt5g5cyZOTk6XfQ9N52kwGMzvvcFgoK2tjb1793LixAlzkkNZWRnV1dX93maVSkVcXBwuLi79/tyWIEHKBtTX1+Pt7T0k0kmvxN3dnbvuugs7OzvzWimtVsu6devQ6XTExsYybdo0yzbSxhQXF5OXl0ddXR2xsbFMnTrVIu0wzbccPXqUkydPcvjwYXJzcykoKCA6OppNmzYxYsQIi7Tt24SFhbF48WJWr17NwYMHycrKumxCgtFoRKfTsXXrVs6ePUtpaSklJSUUFBSYe0gDWWpJpVLh5OTE+PHj8fT0HLDXGUwSpGxAY2Mj/v7+Q+ZDdzmmOSdTiu2mTZuoq6sjKyuL8PBw3NzcpAd1BfX19ZSWlpKfn8/x48c5dOgQdnZ2BAcHk5SUREpKyqC0w2AwcOHCBc6ePUtBQQHnz5+nsLCQ8+fPExwcTFRUFOHh4SQnJ+Pp6clTTz3F+vXrmTdvHiNHjhyUNl4LlUqFm5sb3/3ud3njjTdwd3dnypQpGAwGmpqa2Ldvnzn9u6KiguLiYioqKujp6aG3t5fe3t6LkhxMae4uLi44ODhgb2+Po6MjGo0GFxcXHB0d+9xnSoQwHevg4IBGo8HV1RVXV1ecnJxwcnLC3d2d0NBQ83s7FEiQsgGmOamhnjhhGu5pa2ujtraWkJAQoqOjiY6Oxt3dnc7OTrZv305AQADR0dFDZjjjenR2dtLU1GSeuzt37hxNTU0YjUacnJzw8PBg8eLF+Pn54evrS0BAQL/3VBRFQa/XU11dTVlZGbW1tWi1WmpqamhtbcXZ2Rl7e3tcXFwYP348s2bNws/PDz8/P9zd3XFzc8PZ2ZmVK1eye/duIiIiCAsLs7oEGZVKhaOjI3PmzOGdd95hw4YNlJWV0dHRQXNzM01NTea5Jnt7e0JDQwkKCjKPCJhupkDj4uKCs7Mzzs7OuLm54eDgYC4dZgpApn/b2dlhb29vvt/Ozu6iY02ByxTU3NzchtQXOglSNmC4BCkT0zxFXFwcfn5+tLa2UlVVRV1dHVqtFg8PD5utTn09jEYjPT096HQ6GhoaaGxspKGhgYaGBvN7UlVVhb29Pf7+/oSFhZGQkEBqaip+fn79UmbKtNuyTqejpaWF1tZW2traaGlpoa6ujvr6elpaWmhubqaxsdG8hUtYWBhhYWFERkYycuRIHB0d+1w8jUYj9913H0888QQFBQXEx8cTHx9/w+3tb6befVtbG01NTTQ0NNDZ2UlnZye+vr7mRAWNRoODgwOOjo7mAOLo6IijoyMuLi64u7vj7u6Oi4uLufdjqXlCWyFByoqZLsQtLS3DKgXd9IesKAo9PT2cP3+eU6dOUVtby8KFC4mNjQX+G8yGEtOCTFPZG71eT2dnJxcuXODcuXMcOnSI3NxcqqurcXR0JD4+nuzsbO69915iY2Px8PC4offk6xP/vb29GAwGDAYD3d3d1NfXk5+fT35+PoWFhZSUlFBZWUlYWBjp6ekkJiZy0003MXLkSGJiYq7q9dRqNfHx8UyePJmSkhI2bdpEdHS0VdVvNA3rvfzyy7i7u3PzzTczb968IZM9Z+0kSFm57u5uOjs7cXR0xNXV1dLNGXRffPEFarWajIwMRo8eDXw1/NLS0oKjo+OQq8Kh1+vR6XTk5eVx4MABcnJyyM/Pp7Ozk1GjRjFt2jSeeuopRo0aRUBAQL/vHGtKcDhz5gwHDhwgPz+fs2fPUlxcTE9PD+np6SQlJTFz5kySkpIYPXo0Tk5ON/y6q1at4mc/+xk7duwgKyuLrKwsq/kCUlpayocffkh1dTWffvqp1QXRoU6ClBVTFIXa2lq8vLyG3MX4at18883AV6nAer2euro6/vnPfxIfH09qaqpVTrRfLdMQ2rlz59i/fz/Hjh3j7Nmz1NXVERISQmZmJnfccQdxcXEEBwfj7OxsHkayt7e/4WGi7u5uLly4QF5eHseOHePMmTMUFxej1Wrx8/MjJSWF+Ph4Zs6cSXh4OBEREeYhLNN8i6nC/Y0GFBcXFxYvXszmzZv56U9/yvr16y8aGrSE/Px8Nm7cyJo1a3j33XcvOWQpBpYEKSvX0NCAh4cHzs7Ow+4Pw5ROC1BeXs7p06dpaWlh5syZBAQE2FwdP6PRaB4yO3/+PCUlJZSXl2MwGAgICCA8PJz09HR8fX3NG9b5+Pjg6emJi4vLDa0j6ujooLCwkC+//JKSkhI6Ojro7e3FaDTi6+uLt7c3qampTJ48GU9PTzw8PPD19cXT0xN3d3dzFhkMTJFftVrNqFGjaGho4PDhw3z22WfMnz/fol/OysvLWbt2LQUFBaxYsYK4uDgJUBYgQcqKKYpCU1MT7u7uVpfxNNhM+/x4eXkxevRoHB0daW1tpa6ujoCAAEs37yKKotDV1UV1dbU52aGxsfGixANnZ2e8vLxISEggOjqayMhIgoODb2gIzWg00tHRgVarpb6+3lzFo7i4mJycHLq6usxByN/fnxEjRhAVFWXOSvP397fIfIvpfZg4cSKffvop48ePJygoaNDboigKbW1tbNiwgfLycsLDw5k7dy4uLi4SoCxAgpQVUxSFhoYG3N3d+2Xc35YFBwfj7++PRqOhq6uLpqYmKisr6e7uttog1dDQwK5du8jPz6e4uNi8W+rYsWMZPXo0iYmJxMTE3NDGdKaqBl1dXeZ1OR0dHdTU1HD8+HHOnj1LUVERWq0We3t7XF1dGT9+PFlZWaSlpZmTUKxFREQEd9xxB++88w4nT57ExcUFX1/fQQsOprT606dP88EHHzB16lRuu+22S+5ALQaHBCkrZpqT8vT0HPZByt7e3pwkcOjQIXbu3El0dDQTJ060cMsuTVEUKisrOX78OEFBQdx0002kp6cTHBx83RfcS6XdG41GGhoa2LdvHydOnCA/P59z586h1WqZNGkS48ePZ968ecTFxREdHX2jpzXgTBtmrlq1itdffx03N7dBq5JhClB1dXU8+OCD3HvvvSxcuNCcsCMsQ4KUFTMFKW9v72G9cPXrjEYjer2eH/zgB3h5efV7dlt/UavVjBs3zlxn0LQI80Z7BB0dHZw4cYLjx49z8uRJTp48SUlJCcnJyaSlpXHLLbfw1FNPER8fbw7spsWftsLJyYmHHnqILVu2sHPnThwdHZkwYcKAv67RaOTMmTM8++yzzJ07lwceeIDAwMABf11xZRKkrNyFCxdITEwc9j0pE1M6uru7O93d3VRXV3PkyBGWLl1qVfMFpsB0vUHUNIxXXl5OcXExRUVFnDlzhry8PDw8PAgKCiI0NJTMzEyioqLw9vY2V3Fwc3Mzf6mxpvfkWmg0Gh5++GE++ugjNm/ezKhRo25oWPTbKIrCnj172LRpE87OzjzyyCP4+flZ7Zeg4eSag9Tu3btZvXo1ubm51NTUsHbtWhYtWmR+fPny5bzzzjt9fmbOnDls2rTJ/O/GxkYeffRR1q1bh1qtZsmSJfz+978f8ltRXKuvJ05IkPrvBdfb25vu7m7Ky8spLy+3+d1HTQt4m5qauHDhAnV1ddTW1lJRUUFXVxd6vR69Xo+rqytjx44lICCAkJAQQkJCCA0NJSIiAjs7O5sNSN9kOo/MzEz2799PfX09u3fvZsGCBQP2mseOHWPfvn1UV1czf/58Ro4cOWTeT1t3zUGqvb2d1NRUvvvd77J48eJLHjN37lzeeust87+/mZl27733UlNTw9atW+nt7eWBBx7goYce4r333rvW5gxpX0+cGO7Zfd/U09NDU1MT3d3dzJo1C7CtChS9vb10d3fT0dFBe3s7LS0t5gKlpltxcTHBwcHExMQQHx9vLhJ7o2nQ3d3dGI1G1Gq1VX+uAgICmDhxInv27GHdunVMnTr1hitqfJOiKDQ2NrJhwwaKioqIjIzkjjvuQKVS2cxnaai75iA1b9485s2bd8VjNBoNQUFBl3zszJkzbNq0icOHD5u3Y3799de5+eab+c1vfkNISMi1NmnIUhSF+vp6PD09h+1i3stxd3dnwoQJ5l5IT0+PuQqAtV1cTG007etkNBqpq6ujqKiI48ePk5OTw969e3F2diY6Opr09HQWLVrEtGnT8PT0vKEhJ9Nrf70N5eXltLW14erqSlxcXH+d5oCYNWsWOp2O119/nR07dnDLLbdgb2/fL79jU/mpTz/9lG3btnHzzTezYsUKGbWwMgMyJ7Vz504CAgLw9vZm+vTpvPTSS/j6+gKQk5ODl5eXOUABzJw5E7VazcGDB7ntttsuer7u7m66u7vN/9bpdAPRbKuk1WolceIKOjs7OX/+PO+88w6PPfYYwcHB5ioI1uL8+fOcPHmSI0eOUFlZyb59+4Cv9ilKTU1l4cKF/PrXv8bPzw+NRmP+Ft8fF2JTzb/jx49z+PBh9uzZg9FoZMqUKSxYsMDqg5SDgwNTp05FrVbz5JNPkp2d3W9zRe3t7ezZs4ef//zn/O53v2Py5MkDOu8lrk+//zXPnTuXxYsXExUVxfnz5/nJT37CvHnzyMnJwc7ODq1We9G6Fnt7e3x8fC67bfLLL7/Miy++2G9tVBSFU6dOcfjwYZycnMyLGW8kPbi/mSpft7W14ebmJsUsL0Oj0TBixAieeOIJi0x06/V685qtmpoa6uvrqa+vR6vVmitKdHZ2olar8fHxYdGiRdx9990EBgaaK4m4uLjg5uZmzsS7Xl1dXdTV1XH+/HlOnDjB4cOHqa6uxtXVleDgYEaOHMmtt97KyJEjzZUsrJ1KpcLf35/MzExSUlJ45513uPvuuwkPD7+h521ububEiRP87Gc/4xe/+AXZ2dl4e3tbzd+/+K9+D1JLly41/39ycjIpKSmMHDmSnTt3MmPGjOt6zmeeeYYnnnjC/G+dTnfDH1KVSkVrayslJSWcOnXKPDFtWnHv7+9vnqB2d3cf9BReg8FAa2ureVM0W0ohHkxqtdp8kW9vbzcP+/XXXEtnZyetra20trbS0tJiDkItLS3odDpaW1vp6uoyp3mb0sw1Gg1xcXHmyuQeHh54enqSlpZGaGgo7u7uN1SkVK/X09raSk1NDSUlJZSVlaHVaunp6cHOzg5HR0dGjx5NVlYWXl5e5j2lwsPDzcHcVi7IDg4OBAQEsHTpUv7xj3+QkZFhfj+vR3t7O4cPH2b9+vWkp6czdepUfH19ra4HLr4y4L+V6Oho/Pz8KCoqYsaMGQQFBVFXV9fnGL1eT2Nj42XnsTQaTb9P8Pr4+JgXN1ZWVlJeXk5HRwf+/v74+vri4+ODr68vgYGB+Pj44Orq2udbr+kiM1DBQ6/X09zcjLe3t1VeUHQ6HTqdjo6ODvR6vblkkaen56DWNzOlesNXW5o4ODjg6up6VZ8Xg8GAXq+nu7ub9vZ2Ojs76e7upqenx1x9vqOjg7a2Ntra2mhtbaW+vp4LFy7Q1tZm3ujOwcHB/OXGx8cHHx8f85ccUw2+G8lcNS0yNW2yZ7o1NDRQU1NDeXk5Wq0WnU6Hm5sbI0aMID4+nrS0NCIiIoZE3UdnZ2dmzZrF3/72N44dO4aPjw/p6enXdF6mOai8vDxycnIoKyvjf/7nfwgLC5NUcys24EGqsrKShoYGgoODAcjOzqa5uZnc3FzGjBkDwPbt2zEajWRlZQ10c4CvLmyhoaGEhoaa7zMtnM3Pz6eoqIji4mJyc3M5efIk8FW5lqioKKKiokhISCAlJQUfHx80Gk2fXTRNAeVGLwqmwG1tazVMNelOnDhBbm4uxcXF5iFJ0xYL4eHhFinE2d7ejru7O3Z2dhclCxiNxj7/b6pA3tLSQk1NDYWFhZSWllJbW2seristLTVvJBgYGEhQUBDh4eGMHTuWyMhIwsPDCQ0N7felE6Z2mvZy6u3tRafTUVRURE5ODocPH6agoIDe3l5GjBhBdnY29913H8nJyfj5+Q3JHoFarcbT05Pvfe97vPvuu6hUKhITE686ochUraOxsZF//OMftLW1cfvttzN9+vSBbLboByrlGrc4bWtro6ioCID09HReffVVbrrpJvM3yBdffJElS5YQFBTE+fPneeqpp2htbeXUqVPmb7fz5s2jtraWP//5z+YU9LFjx151CrpOp8PT05OWlhY8PDyu8ZQv7Upvg1arpaioyLxN9+nTpzl8+DAqlYrg4GBiY2MZNWoU6enppKWl4e/vf8PZeA0NDezYsYN///vfvPHGG1aT9djT08OPf/xjPvjgAyIiIhg9ejQhISFUVFSwfv16YmNjufvuu1m5cqVFCoOa6PV6ysvLaWlpMa+nMt1KS0spKyuju7sbZ2dnwsLCGDFiBGFhYYSHhxMZGUlUVBQjR47Ezc3tW3vL/R2Mu7u7qaioYMeOHeTk5HDs2DG0Wi3+/v6MHz+eSZMmMW7cOCIjIy/5ObP1XtOVKIrCk08+SWtrK1OmTOHee++96p/r7e3lnnvuwdnZmXnz5nH33XcP6ffK2l3tdfyag9TOnTu56aabLrp/2bJl/OlPf2LRokUcO3aM5uZmQkJCmD17Nr/4xS/6lBdpbGxk1apVfRbz/uEPf7jqb6QDEaSu5Ou7pPb29prXuDQ1NVFbW0tlZSWlpaUUFhZy7tw5DAYDXl5exMTEkJGRQXJyMiNHjsTX1/eqL9xarZa1a9dy/Phxfv7zn1tFeZampiZ27tzJ8uXL+dnPfsbUqVPNC0kNBgPHjh1j9erVGAwGli1bxv333z/obTx+/Djr16/n//2//wdAUlISAQEB5u0vgoKCCAkJISAgAFdXVxwdHbG3tzeXEDL917RX0kCvl9Hr9VRVVXHgwAGOHz9Ofn4+FRUVODk5kZGRQVJSErGxsYSFheHh4WEe+tZoNDecaGGrCgoKeO+99zh06BB//etfCQsL+9bfkU6n41e/+hWnTp3if/7nf8jKyho2O11bq6u9jl/zuMC0adOu2OvYvHnztz6Hj4+PTS3cvVx5G39/f0JDQ4mNjaW5uZnGxkYaGhpoamoyz9eUlZVx9uxZOjs7zVmMpg3kgoODCQwMxM/P76JJdNNwn4+Pj9UM9zU1NbFlyxbCw8OZNGkS8fHxfXYLHjt2LJmZmRw8eJBt27ZZJEjt3r2bnTt3EhMTw7Jly/D19cXFxcV8c3V1xd3d3ZyQMpjvrdFopLW1laKiIkpLSyktLaW8vBydToe7uzseHh5MnDgRb29vc0A1zXPdaKLFUBIREUFqaipVVVW8/fbb/OQnP7ni77Gqqopdu3Zx5MgRvv/975OcnCzVbWzI0Bu8HkROTk44OTmZ14DBVxciUxZYVVUVZWVllJeXU1NTQ3t7O3q9nq6uLurr68211kwXKC8vLzw8PHB3dzcHu8DAwAH5tmwwGMzzNJfzzaKobW1tHDt2jMTERMLCwi7azt7b25tRo0Zx9uxZ8vLyzJl2gzWkUltby9mzZ+nq6mL58uXcc889VjOcYzAY0Ol0fP755zQ1NZlvOp0OjUZDZGQkMTExREVFER4ejre3t6WbbLWcnZ0ZPXo09fX1fPLJJ9x3332EhoZecpSioaGBEydOsG3bNuLj480LpK3lcyG+nQSpfqZWq/H29sbb27vPQklTSnllZSVnzpwhPz+fI0eOUFpaSl1dHX5+fubtFMLDw1EUhZqaGoKDg7lw4UKfxcyXY0oUuBrd3d3o9XpzFYRvsrOzw9PTE39/f/N9XV1dlJWVMW3atMt+qw8ODsbT09OcATcYF1vTOR88eJDW1lZSUlIs0ou7kt7eXmpra1m9ejXjxo0jJSWFiRMnkpiYeMPLKYajmJgYjEYjH374IVu2bGHJkiUX7TvV09PD8ePH2bFjB2VlZfzrX//Cy8trWA6R2jIJUoPEzs4OLy8v886yJqb1LmfOnCE3N5f8/Hx27txJWVkZjY2NrFu3jpdeeumK3/xMk8Lt7e1X3R4nJyccHR0vO0zi5eXFPffcwwsvvGAOSEajkc7OTry9vS/7h27aalxRFDo6OvDy8hqUb60Gg4G//OUvjBkzhgULFljd4mcnJyfi4+M5deoUMLSTGwaDWq0mPDycX/ziF9x9992kpaXh5uaGk5OT+UvLsWPH+POf/4xKpeL111+3muQjcW0kSFmYqccybtw4MjIyMBgMdHZ2cvDgQb773e+yY8eOq8owc3BwuKaMQtNF8nIXS5VKZU4qMDGthaqtrcVgMFzy50xDWPb29oM2rGIwGPjggw9Qq9WkpqaSlpY24K8pLM/FxYUxY8Zw66238u9//5uOjg5uuukmjEYj58+f5/HHH2fGjBksWLCA+Ph4SzdXXCcJUhZmyh5Tq9XmHotprU5SUhIxMTFXNa9jWtQ6kEHB2dmZ2NhYTp06RXd39yWrjpeUlNDY2Eh4ePigFMU1Go10dHTw9ttvm3dRtdZ1QtJ76l+mL1IPPvggL774IocOHSIkJAQvLy9eeOEFJk2axOzZs0lMTLSa5CNx7azzr3mYq6yspKysjPT0dJydna1mDN3Dw4MJEybw9ttvk5+fj4uLCz4+PubHq6urOXXqFB0dHYwbN25QgkV7ezvHjh2jra2NMWPGmBeNi+FBpVKRlJREUlIS1dXVrFmzBj8/P1pbW5k5cyaJiYmSam7jrOPqJ8wURTEvOM3Ozraqb9/e3t7Mnj2b3t5eNm/ezOHDhykrK6OqqorS0lJ27NhBXl4ebm5uzJ49e8DbY9ry4vPPP2fcuHHExMQMyro5YT1UKhVOTk4sXrwYvV7Pu+++y9tvv82MGTPIzs7uk/gjbJP0pKxMT0+PuWDo5MmTLd2cPtzc3MjMzOTZZ5/llVdeYc2aNURERBAaGkpFRQXHjh1jzpw5LF261LwR4UAybdPx0Ucf8eWXX8o2C8OQKUkiKSmJ0NBQ7O3tcXR05NZbb0Wj0ZizV2UTQ9t1zRUnrMFgV5wYTAcOHODzzz+nsbGRP/3pT4B1zWWYinRWVFRw8uRJioqKaGlpwdvbmwkTJhATE2NO8x3odm/atInNmzejUqlYvXr1oLymsC6mLNIf//jH1NbWMmrUKO677z5++ctfUlVVxciRI8nIyGDixImkpKRYurniawas4oQYWLm5uQCMGTPGKi+4KpUKe3t7QkND8fLyIjMzE4PBgL29Pe7u7jg5OQ3KJHV9fT1Hjx7l/Pnz/OIXv5AANQyZ1h4+9dRTdHd3m3czDggI4Pnnn2fv3r3s2bOH9957j7feeovExEQSExNJTk4mMTGRqKgo+czYAAlSVsLUoT116hR+fn591lJZI0dHR4uuRdq3bx8NDQ3ExcURExNjsXYIy9Dr9Wi1Wv7zn//Q2NjIggULmDBhAuHh4ahUKmJjY7Gzs8Pb25vc3FxycnLIyckhLy+PAwcOMHLkSOLi4ggKCiIiIoLY2FhcXFysJklJ/JcEKSui0+moqKggNDRULryXoSgK7e3tbN++HY1Gw7Rp0y4qzySGNlNR3kOHDrFhwwZmzpzJ3Llz8ff379Mzio6OJjIykuTkZMLCwlizZg0nT54kLy8PjUZj3sEgJSWFyZMnExQUhKurKy4uLuZyZNLTsjwJUlbk1KlTODo6EhgYaLNZSd/cw8loNJo3h+yvP/hTp06Rl5fHvHnzrnu3Z2F7TKMNDQ0NrF27lk8//ZRp06bxxBNPXHaI2c7OjpEjRxIZGcmsWbN49dVX2bJlC+Xl5RQUFHD27FnWrVtHUFAQUVFRxMXFkZSUxLJly8x7k5n2ipOAZRmSOGEljEYjzz33HHZ2dsyYMYNp06ZZuknXxWg0UllZyeHDh837Ia1evZrx48fj4uJyw89vMBhYsGABU6ZMYdasWde8O6uwXaaNSX/0ox/R1tbG9OnTefTRR68qc890mVMUhZ07d/LFF1/wxRdfUFBQcNGxarWa4OBgUlNTiYuLIyMjg1mzZhEYGCiftX4kiRM2RlEUduzYwapVq8zb2tsKvV7P6dOn2b17NydPnqSgoICSkhLa2tro6uriyJEjhISEkJCQcEOv097ezsGDB6mpqWHixIkkJCTIRWOYMM1BPfTQQ8TGxnLnnXcyZcqUq55D+noZsPHjx5OYmMjixYt56623WLt2LS0tLeZSX0ajEa1WS3NzM/v37+eTTz7hD3/4A6NHjyY1NZUJEyYQGxsrleoHiQQpK9Db20tNTQ3Nzc2Eh4fj5+dn6SZdtZ6eHmpra3n11Vc5c+YMtbW1NDU10dbWZj6msrKShoaGG3odRVFobm7mvffeY+7cuURERAxK2SVheZ2dnZSVlfHGG28QFhbGnDlzSEtLu+4g4eLigpOTE25ubjz44INER0ezdetWzpw5Q11dnXmZhalgs06n48KFC9TU1HDy5En27NlDREQEERERjBgxgtTUVEJCQnBycurP0xb/PwlSVqC7u5szZ87g5+dn3qTPViiKQnd3N0ePHqWgoAC9Xn/RMXV1dTQ3N9/Q67S2tlJSUsLx48f5/e9/j4+Pj/SihoHOzk6KiorYvXs3Z86c4ZFHHiEzM/OGv8ip1Wrc3d0ZP348wcHBeHt7s3v3bo4ePUplZSUdHR3mY41GI11dXVRVVVFVVcXJkyfx9PQkODiY+Ph4SkpKGD16NPHx8URGRt7oKYtvkCBlBTo6Oti9ezeTJk2yuTk2jUZDUFAQ06dPp7S09JJBqr6+npaWlut+DUVRKC0tZe/evQQFBTF27Fg0Gs2NNFvYAL1eT1lZGRs3bmTNmjU89thjLFiwoF93KFapVIwYMYJHHnmE8ePHs3nzZj755BPzpp2X2m9Nr9fT0NBAQ0MDp0+fZt26daSkpPDQQw+xYsWKfmub+IosCrAC7e3tbNq0iVmzZtlkaR9nZ2dWrlxJcHDwJS8gNTU1NDY2Xvfz9/T0cOjQIT777DOeffZZ2UZ9mMjPz+e5554jJyeHn/70pyxdunRAf/dpaWn8+Mc/5osvvuDBBx8kKCjoql6vp6eHsLAwm83ItXbSk7KwtrY2Kisr0Wq1ZGRk2OSaH7VazciRI5k9ezbr1q2joqKiz+PV1dXU1tbS2dl5XfNImzZtorCwkLFjx1ptJQ7Rf3p7ezl27BirVq1ixowZzJs3j8zMzAH/vZu2u/Hz8+OXv/wlCxcu5PPPP2f79u3k5+df9ufc3Ny45ZZbGDt27CUf7+np4cKFC2zfvp2amhrs7OwICAggISHhsj8j/kuClIVptVrOnj1LRkYGTk5ONnkBNpVKWrhwIWfPnqWurq7Pdvft7e1UVlZy/vz5a6qkYarLtmPHDgwGA4sWLbK6HXdF/2ptbeXs2bP83//9H/PmzWPOnDkkJiYOWlKCKVB5enoyduxYgoKCGD9+PFu2bGHTpk1cuHDhoiHAjIwMRo0adcl5stLSUg4cOMDHH3+MwWDA39+f3t5ec+br+PHjWbVqlXlNlriYBCkL02q1nDt3jqysLJtfMJiWlkZiYiKFhYV9elN6vZ76+nrKysquudzT0aNHaWpqIiYmRnbcHeKamprIz89n48aNODo6Mm/ePBITE/H09LRIe7y9vfHy8iIoKAgfHx88PDzYuXMnFRUVtLa2YjQaUavVTJw48ZLZfW1tbRw8eJCNGzdSU1PD9OnTCQsLM2fEFhYW8tlnnzFmzBiysrL67M0m/kuClAUZjUZqamooKCjg8ccft/m6YQEBAYwZM4Zz585RVVXV5xtnc3MzVVVVV/1ciqKg1+v5+OOPCQoKIiMjA19f34FotrACnZ2d5OXlsWXLFrZt28arr75KRkaGxXvOKpUKf39/5syZw9ixYwkMDGTLli0UFBSg0+nQaDTMmDHjkunwFRUVbNmyhaNHj7JixQrzwmP4anTh7Nmz3HHHHbz77ruEhITg7e1t019SB4ptXxVtXFtbG1VVVWi1WiZMmGDzQQpg5syZzJw586LdUFtaWqiurr7q5zFtB/LFF18wbtw4q9tbS/QPUxmt/fv388c//pHjx4/zxz/+kezsbIsHqK9Tq9X4+/vz3HPP8eqrr/Loo4+SkZHBhAkTyM7OvmRvb/PmzZSVlREfH89DDz3U5zFXV1fi4uL43ve+x8aNGykvL6e3t3ewTsemSE/Kgg4fPkxLSwvjx48fMinVoaGhjBs3jmnTpvHZZ5+Z729paaGysvKqn6e1tZUXX3yRe+65h5SUFNkCfIgyGAx89tln/OpXv2L+/PncfvvtxMXFWXWPIj09naSkJB555BGqqqou+7d75swZ7O3tiYuLu+Qxjo6OTJgwAYPBQElJCVqtloiIiIFuvs2x/a/uNuzkyZO0t7ebM5es+Q/zaqnVamJiYli0aFGfTEWdTkdNTQ09PT18W7nI1tZWzp07R25uLgsXLiQkJGRIvDeir8bGRtavX88f/vAHli9fzm233caIESOsPoHAzs4OJycnvLy8iImJuewISHt7O/b29nh4eFzy79vUO1OpVHR2dtLV1TUYzbc5EqQsQFEUent7KSoqore3l+TkZEs3qV/5+vqSlpZGfHy8+Q+4o6ODurq6q+pNVVdXs3PnTpKSkoiOjrapChzi6tTU1HDgwAHWrVvH2LFjuemmm4iOjraZUlemLEAXF5fLfoFSq9XmudXL6e3tRVEU1Gr1kBjuHwjyrlhIY2MjdXV12NvbD7m9o1xcXAgPD2fatGnmeYXe3l4aGxspKCi4Yk+qo6ODwsJCdu3axV133YWbm5v88Vox05xSa2sr7e3t9PT0fOvPNDY2kpuby5dffolWq+XRRx8lJiZmyH0Z8fDwQK/X09zcjNFovOhzbzAY0Gq1ALi7uw+58+8v8tdvIQcOHMDLy4uoqCibXMD7bdzd3Vm2bBk+Pj7Y23819dnV1cWZM2cuG6QURaGgoIDjx4/T2dnZ7yVwRP8x7RlmMBjo6uoiJyeHkydPmi+6l/sZg8HA559/zj//+U/Onz/PG2+8QWRk5JD8PcfExGAwGCgtLe2zbhD+O5py7NgxHBwcCA0NtanC0oNJgpSFbN68maioKKvfJv56OTg4MHr0aGbPnm1Oz+3u7qa4uPiyP6MoCmvWrOHcuXM8/fTTVj83MZw1Nzdz+PBhXnzxRZKSknj++ec5ePBgn+r336QoCn/4wx946623SE9P589//vOQLsg6f/58RowYQV5eHn/729/6PNbV1UVJSQl//OMfmT9/PlFRUUMyUPcHye4bZKYx6v379/OjH/2IuLg4SzdpQJjG6e+55x7y8/NpaWmht7eX8vLyy/aktm3bRm1tLWFhYUycOFGSJayIqReUm5vLli1bOHHiBDqdjsTERH77298SHx+Pv7//JbMwFUWhsbGRt956i40bN7Jq1SpzJfOh/DsOCwtjyZIl2NnZ8dprr3HixAni4uLo6emhoqKC06dPExISwsqVKxkxYsSQfi9uhAQpCzAYDEyZMoWEhIQhv3FaamoqycnJ1NTUUF9fT3V1NQaDoU91DVPg/vzzz/H29iYrK0tSzq1Ec3MzlZWVFBYWcurUKZqamnByciIjIwN/f3+io6NJTk7G19f3kj1fRVEoKSlh9+7d7Nu3j8WLF5OVlUVwcLB5GHio0mg0pKWlYW9vj4+PDw0NDVRUVJgrVYwdO5bx48cTHx8v81FXMLQ/JVZsyZIlREVFDZn1UZeiUqkICAhg3LhxnD9/ntraWrRaLTqdDh8fH/NFzWg0UlpaSn5+PnfccQcZGRkWbvnwZfrC0NjYSE1NDWVlZZSVlVFcXMz58+fN6+DGjBlDVFTUZRfcmnrLFRUV5OTksHv3bgICAli6dCleXl7DZijXtFdVXFwce/fupb6+HrVajY+PDzExMUNq+clAkSA1yFQqFU5OTkydOtXSTRk006ZNo6ioiCNHjlBfX8/Zs2cZM2YMLi4u5gnk999/n8jISJKTkwkPD7d0k4cdg8GAwWCgu7ub5uZmdu3axccff0xhYSEhISFMnTqV73//+yQkJHzrBdUUoFpbW/nggw84cuQI7u7urF69Gnd392F3QXZyciIyMnJIz78NJAlSYsDFxcUxadIkTp48yZdffklubi6JiYm4uLig1+upq6vjzTff5OOPPyYlJcXSzR2WiouL2bZtG9u2bePQoUMkJCSwfPlypk2bRmBg4DX3fLq6urj11lvR6/UsWrSIlStXypCWuC4SpAZBZ2cnOp0Og8GAq6urxao6W4pKpSI1NZUlS5awf/9+SktLzetpKioqePvtt7nllluIi4vDzc3Nwq0dHnp7e6mtreXQoUP85z//4dy5c4wePZrp06fz9NNPExkZiZubGxqN5qqr8xsMBgoLC/n888/56KOP0Gq1vPLKK8yePRtnZ+dh14MS/UOC1ADp7u6mrq6ODz/8kPz8fNra2jAajWg0GkaMGMHixYuJi4sbNgkCAQEBpKWlkZKSQnV1tXlPnfPnz7Njxw6ef/55PDw8ZOHuADJV3c/NzeXEiRNUVVVhb2/PqFGjuOWWWwgLCyMkJITg4OCrXrunKApNTU0cOnSIY8eOkZ+fT3FxMb6+vvzgBz9gypQp+Pr6yu/1G06fPs3mzZv5zne+g5+fn7w/VyBBagDo9XpqamrYsGED27ZtM89DqVQqmpqaqKysxGg0snjxYhITE4dF78HFxYWwsDCmTJnC/v370ev1lJWVcebMGTQaDePGjRvSSSSWYqp4UFtbS2VlJcXFxVRVVVFXVwdAeHg4U6ZMIT09HY1Gc1W9HUVRaGtr48KFC1RVVVFQUEBpaSlarRZFURg1ahQZGRnccccduLq6ygX4Eurr69mzZw9TpkzBx8dH3qMrkCA1AHQ6HUeOHOHll19mwoQJ/PCHPyQuLg4HBwcqKyv529/+xp///GdcXV1xdnYesgt6v8nT05P58+ezbt062tvbOXz4MCdOnOC2227D09NThoP6iSkZpaOjg6amJs6cOcOhQ4fYsWMHbW1tzJ49m6VLl5KamnrVe3SZ1kl1dXXR3t5OcXExJ06cYPfu3Zw4cYLp06dz6623kp6eLpW8r4JGo8HLy4szZ86QkpIy5NPxb4S8MwPg4MGDrFu3DkVR+NOf/oSnp6d54tm0+DE/P59Nmzbh6upKUlLSsLhAu7q6Mm7cOEaMGMGhQ4fYvXs3nZ2dLF++3NJNs3lfXyDd1dXF6dOneeutt9iyZQve3t5kZmby1FNPMWfOnKueY/r6c/b29lJTU8OHH37I+++/T3d3N7GxsSxatIi//vWvuLi4SG/gGpjS0k+cOMGSJUtkFOEKJEgNgKKiIioqKpg5cyZeXl6XvCAsXryYN954g6KiIjo7O4dN5pOdnR2PPvoor732GuHh4SxatGjYnPtAMRV43b9/Pzt27GDfvn10dXWxcOFC3n33XSIjI/Hw8MDBweGqs/RMvbGDBw+yb98+cnJyyMvLY+bMmfziF78gMTGRgIAAHB0drWpzQlvh7e1NQkICr7/+OgaDwdLNsWoSpAaAqSK0aR+kSwWp8PBwHBwcaGtro7m5eVhcqI1GIx0dHezZs4fq6mr8/Py+dW8pcXmtra2UlZWxfft29u7di5ubG8HBwdx///3Ex8cTGhpKYGAgzs7OVz2c1N7eTnV1NXv27GH79u0oioKvry9Tpkzh+9//PuHh4YSEhODu7i7B6QZ4eHgQFRXFuXPnzNt1DIfRlOshQWoA6PV6DAbDFf+IHR0dUalUGI3GYbFtdEdHh3mfqIqKCrKzs1Gr1Rw/fhyNRsO0adNkyOhbKIpCR0cH58+fp6ysjJqaGi5cuEBTUxNBQUFERUURHx9PQkICUVFRAFe18NZgMFBRUUFpaSnl5eVUVVXR0tKCp6cnISEhREdHk5CQQFJSEg4ODle9mLe6upry8nK8vLyIi4sbNlUmroZGo8HHx8dc19CU7i8uJkFqANjb22NnZ0dnZ+dlj+no6MBoNGJnZzdkv5F+vfJASUkJBw8e5J///CcTJkzgzjvv5PDhw+zatYszZ87g5eXFqFGjcHd3l2rQ39DT00NnZyctLS1UVlayb98+cnNzqa+vx8fHhzlz5rBgwQK8vb2v+r3T6/V0dXXR1tZGfX09ubm5HD58mLKyMvR6PbNnz+b73/8+ISEhODk5XdVzmipWmEYHjh07xokTJ8ybV0qQ+i/ThonBwcGUl5fj7+8vQeoyJEgNAG9vbzw8PCgvLwfo05U3XbjLy8vR6/V4eHgMyX1kTOdpMBjYtGkTa9asobCwkKeeeoo77rgDtVpNRkYGs2bN4q233uKuu+7ipZdeYsaMGYSFhUk9s6+pqKhg//79fPzxx+Tk5DBv3jwWLVpEZmYm0dHRV/Ucpt+HaZNCrVbLqVOn2Lx5Mx988AGjRo1i3rx5LFu2jPT09KsKdl8fqlUUhebmZnNC0Icffoinpydz584lNDT0+k58iLO3tyc5OZlz584RFxeHl5eXpZtklSRIDYBRo0Zx6tQpPvroI8rLywkNDe0zJ6DX63n33XcJCgoiOTl5SKafGo1GGhsbefzxx6moqGDGjBn88pe/7LMlgUqlIjo6mp/+9KfMmTOHZ555hvXr1zN37lzuu+8+m9lKfKDpdDpUKhW33XYb/+///T/c3d2xt7e/piBuNBppbW3liy++YN26dZw7dw4XFxdmzJjBrl27CA8Px9HREbVafU3P29vby969e/nwww85fPgwarWaSZMm8de//pXMzEw0Go1sjX4ZDg4OjBkzhtOnTzN58mSpWXkZQ+/qaAVSUlJoampi586dPP744yxfvpyRI0fi4OBATU0Nn332GZWVlfzwhz9k6tSpQ67H0NDQwPHjx/ntb39LWFgYTz/9NMnJyRfVgDP1ljQaDenp6fzud79j06ZN7Ny5k0OHDvH0008TFRU17IeJ4uLiCA8PR1EUvL29rymQlJaWcuzYMQ4ePMihQ4cICAhg7NixfOc73yE8PBwfHx/8/f3Nc6RXo7Gxkby8PA4dOsSePXtQqVSMGTOGOXPmEBERgZ+fHz4+Pri6ug65z3Z/Mm0Mum7duituFjncqRQbTK/S6XR4enrS0tKCh4eHpZtzEdNE9Jo1a9ixYwd+fn7m1fw9PT00NjYSExPDXXfdRWJi4pDZPt5oNJKXl8eRI0c4cuQIAPPmzWP8+PFXtaq+u7ub48ePc+TIEU6dOoWLiwsLFy4kJSVlyO+71V/0ej3t7e2cO3eOo0ePUlVVRU9PDyqVCkdHR2JiYkhKSiIyMtIc8L6Noih0d3dTWVnJ0aNHKSgooKOjA0VRsLOzY8SIEYwePZro6Gh8fX2H5MjAQDAYDFRVVXHzzTfzu9/9jptuumlYvXdXex0fPu/IILKzsyMsLIzly5djZ2fHgQMHzHNQnp6epKamsmzZMkJDQy8a0mptbTVfVHx8fCx0BtdGURR6enooKipi+/btHD9+nM7OTh577DEyMzOvuiek0WjIysoiIiKCgIAA3nzzTRwcHNDr9YwaNQp/f/9h9Ud8tUxZf42NjTQ0NFBdXc2pU6c4ePAgdnZ2JCYmkpmZyYQJE/D29r7qhbzd3d00NjbS1NREfX09hYWFHD58mOrqaiIjIxk7dixTpkwhOjpaekzXQa1WExwcTE9PDzqdjs7OzmFTy/NaSE/Kyhw6dIiysjIcHByYN2+eOfPPWi8CpkWf5eXl/PjHP6azs5OpU6fyve99j4CAgOt+Xr1eT3FxMStXrsTR0ZFZs2Zx9913m4txWuv7MVgURcFoNGI0GtHr9eYkiL1793L+/Hmys7O5//77yczMxMPD46oDk+l59Xo958+fZ/PmzezcuZO8vDxCQ0P5zne+wy233IK/v79kYfYDRVGYO3cut99+OzfddBMxMTGWbtKgudrr+DXNZr788suMGzcOd3d3AgICWLRoEQUFBX2O6erqYuXKlfj6+uLm5saSJUuora3tc0x5eTm33HILLi4uBAQE8OSTT6LX66+lKUOWv78/Z8+e5Ve/+hUPPPAADQ0NGI1GSzfrsmpra/nkk0+YPHkysbGx/OY3v+Hpp5/G39//hp7Xzs6O2NhY1q9fz1133UVOTg633XYb+/bts+r3Y7DodDqOHj3Kb37zG8aPH88DDzxAfX09P/jBDzh48CDvvPMOM2fOvKYvcabA9M9//pMFCxYwb948jhw5wsKFC81zhd/73veGxdbvgyk1NZX6+noqKios3RSrdE09qblz57J06VLGjRuHXq/nJz/5CadPnyY/P988r/LII4/wxRdf8Pbbb+Pp6cmqVatQq9Xs27cP+GocNi0tjaCgIFavXk1NTQ33338/K1as4P/+7/+uqh1DuSfV29tLY2MjBQUFrFmzhmPHjvH0008zduzYG+qZ9Leenh62bNnCpk2bOHXqFCtXrmTSpEn4+vpe0yT8lXx9nVVRURF79+7lX//6F/Pnz2f58uWEhoYOm6QKvV5PS0sLH3zwAevWraOurg5PT0/S09OZOnUqSUlJuLu7m4sWX+37YjAYKC0tZePGjezcuZOmpiZCQkKYMmUKEyZMwMvLCzc3N5ydnYfsej5LUhSFf/7zn5w4cYIxY8Zw7733WrpJg+Zqr+M3NNxXX19PQEAAu3btYsqUKbS0tODv7897773H7bffDsDZs2dJTEwkJyeH8ePHs3HjRm699Vaqq6sJDAwE4M9//jNPP/009fX1V/WHMJSDFHyVgKDT6cjLy2Pjxo1otVqSkpIYP348mZmZFh3uUhTFvE9WYWEhdnZ2pKWlMWfOHAICAgYsaLS1tVFRUcH27dvJyckhKiqKSZMmkZmZOSSTKkzzTEVFRZw/f57CwkLOnj1LaWmpeRPJ5ORk4uLiiI2Nxd/f/6rTvHt7e6mqqqKwsJBTp05x7tw5PDw88PLyIiAggPDwcGJiYoiMjLzqYrTi+iiKwv79+/n73//O6NGj+dGPfmTpJg2aQUmcaGlpATBP8Ofm5tLb28vMmTPNxyQkJBAREWEOUjk5OeZ0ZJM5c+bwyCOPkJeXR3p6+o00aUhQq9V4eXmRnZ2Ni4sLf//73zl9+jTt7e0YDAbS09NxcXEZ9ItHd3c3Wq2W7du3c+DAAfz9/cnKymL+/PkDnm7s5uZGXFwcQUFB5u0n9Ho9nZ2djBs3zlwn0ZZ9vVpDQ0MD5eXlVFZWUl1dTXV1NTU1NQQHB5OcnMyMGTMYNWrUVe9FptfraW1tpaKigqqqKqqqqqisrDQn9GRkZJCVlcWIESOG3c7RlhYZGWlOfOnt7ZW5vm+47iBlNBp5/PHHmThxonk/JK1Wi6Oj40UrpwMDA9FqteZjvh6gTI+bHruU7u5uuru7zf/W6XTX22ybolarSU9P5xe/+AXvv/8+27ZtY9++fbz00kvEx8df07DOjTBl71VVVbFhwwZee+01vve977FgwQJGjRo14K9vYmdnh5eXF88++yyffvopa9eu5Xe/+x3Lli1jyZIluLm52VRShdFoxGAw0NvbS29vL1qtlqKiIk6dOsXhw4c5duwYKSkppKenc8cdd5CcnExUVNQ1JUGYnru5uZmzZ8/yn//8h127duHn52femHDWrFnSY7IgU0WO1tZWWlpahmQFmhtx3UFq5cqVnD59mr179/Zney7p5Zdf5sUXXxzw17FW3t7e/OAHP2Du3Ll89NFHzJs3j5/97Gfceuutg7LBXEdHB59//jkfffQRJSUlfPLJJyQmJlpkjkKlUqEoCosWLWLixIns27ePn/3sZ2zcuJGXXnqJESNG2EwNtJaWFoqKivjyyy/ZuHEjJSUlBAUFkZmZyd13381bb7110c621xJIWltb2bp1K5s3b+bAgQO0tbVx55138uGHHxIVFdVnfZ4EKMswfZ6jo6Pp7e3l7NmzTJo0ydLNsirXFaRWrVrF+vXr2b17N2FhYeb7g4KC6Onpobm5uU9vqra2lqCgIPMxhw4d6vN8puw/0zHf9Mwzz/DEE0+Y/63T6YZVCRHTBzkyMpIf/OAHTJw4kZdeeoldu3Yxc+ZMli5dOiDrKxRF4fTp0/zf//0fOp2OtLQ0fv/735uzuyx1YTO9rq+vL3PmzCEtLY1XX32VBx98kNtuu40FCxYQGxtrkbZdjqk3evz4cbZu3UpOTg5arRYnJycyMzN59NFHGT16ND4+Pjg5OZn3abrWGoYXLlzgzJkzfPbZZ3zxxRckJCQwbtw47rrrLlJSUnB2dsbJyUl6TlZEpVKRmJhIfX09jY2Nlm6O1bmmxAlFUXj00UdZu3YtO3fuvOhCYEqceP/991myZAkABQUFJCQkXJQ4UVNTY85W++tf/8qTTz5JXV3dVX0LHuqJE1diqsF26NAh9u3bR1VVFR4eHqxYsYLY2Nh+Gf5TFIW2tjZ27tzJO++8w4gRI0hLSyMjI8O85YK1XOAURUGv15Ofn8/WrVs5e/YsGo2GWbNmsWDBAosVqjUthj137hxnz56loKCAoqIi7OzsCAgIIDAwkICAAAICAvD39ycoKAhvb+9rzow07dF1+PBh86Lx7u5uwsLCiI2NJTw8nODgYPz9/c0JJtbyuxP/Zfq9ubi4WHVBXoPBQHNzM+3t7bi7u192U9erMSCJEytXruS9997js88+w93d3TyH5OnpibOzM56enjz44IM88cQT+Pj44OHhwaOPPkp2djbjx48HYPbs2YwaNYrvfOc7/PrXv0ar1fLcc8+xcuVKmxmmsSS1Wo2npydTp07F0dGR/fv3k5eXx8cff8zcuXNJSEjAzc3tuj84PT09VFZWcuLECXbs2IG7uztTp04lPT29T6/ZWqhUKhwcHEhNTTVvkZKfn8+XX36Jo6Mj2dnZ5oKsA627u5vm5mYuXLhAdXU1FRUV5ooNLS0tODo6EhYWxqhRo0hMTCQ0NPS6MxNNWZanTp2ipKSE+vp6amtrcXR0JCQkhDFjxjBhwgTc3d2HTZq+LRuMYftr1dHRQVtbGy0tLeZqJk1NTbS1tZlHAAajcvs1/eX+6U9/AmDatGl97n/rrbdYvnw5AL/73e9Qq9UsWbKE7u5u5syZw5tvvmk+1s7OjvXr1/PII4+QnZ2Nq6sry5Yt4+c///mNnckw4+joyNSpUxk5ciR79+5l9erV6HQ6br/9duLi4vDw8Limi9PX67Pt2LGDDRs2ALB69WrCw8Nt4gtEUlISYWFhHDp0iL///e+8+eabODk5kZiYiI+PT7+fgynpoauri46ODhoaGigpKeHMmTPk5uZy5swZEhMTSUlJYcKECWRmZuLn59cvPRmj0UhxcTH//Oc/OX78ONOnT2fWrFmkpKRY5QVPWC9ThZHe3l56enro7u6mp6eHuro6ampqKCsro7i4mIKCAqqrq3FycmL06NGDNqQuZZGGAL1eT319PY888gitra3MmzeP++67z5w1eaWL4td//Xv37uWVV16hvb2dWbNm8dRTT1l07ul6mdYY/f3vf+eNN97gnnvuYeHChaSlpZmPudZzutSfSUNDA0VFRezYsYNNmzZRXFxMcHAw6enpTJkyhfnz55szDgdCZWUlJ0+eZOLEibi7u8t2GDbiWi65/fW3d6XXbG1tpby83Lxu7vDhwxw9ehRFUQgICCA2NpbRo0eTlpZGeno6/v7+/bKNzqAs5rUUCVJ9mdKNu7q6+PTTT1m/fj1lZWX86Ec/YtGiRVe8eJkWDr/55pv885//ZNmyZcyZM4ekpCSrrxt4OaaPdG9vL0eOHOHNN9+kqqqKrKwsnn/+eZydna8rSHV1dXHs2DG2b9/O3r17qampwdHRkUmTJjF9+nTS0tLw8vLCwcEBOzs78xDjQL1/ptp9ph6zrf2ehqtz585x4sQJjh8/bq7wMX78eBITE3FwcODVV19lwoQJTJkypd+G0xRFwWAwUFZWZl4YfurUKU6ePGleFhQREUF0dDSJiYmMGTOG0NBQXFxczPuB2dnZmZd49MdnTYLUMGP6NV64cIGzZ89y6NAhtmzZwuTJk1m2bNkl663pdDpyc3P5+OOP0Wq13HPPPaSnpxMUFDRktg8xlVQ6dOiQebnEM888Q1RU1BW/DZqGP0+fPk1eXh5nzpzh7Nmz5kysyMhIQkJCzAkPPj4+17UZoRheNmzYwEcffURlZSVRUVH4+PhQX19PUFAQU6ZMYezYsfztb39j7NixZGZmXtfCakVRaGxspLi4mNLSUvOtuLgYZ2dnvLy88Pb2xtfXl4iICEJDQ83ltFxcXHB1dcXDwwONRjOg85myVccwY7ow+vv7o9Fo8PDwoLOzk+LiYj766CMyMzMZPXq0eaK+pKSEffv2cfLkSbq6upg7dy7Tpk3Dy8trSBUPdXd3JykpCRcXFxwdHdm9ezf//Oc/mTZtGqNHjyY0NNT83nV1dVFfX49Wq6WyspKioiJ0Oh0GgwGVSkVCQgL+/v4kJSUxYsQIgoKCZMtvcU02btxITU0NERERzJo1Cy8vLy5cuIBGoyEkJAQXFxcmTZpESEjIt86hmpY11NbWmtPXGxsbqauro7293Ty31Nvbi5ubG6mpqebg5OfnR0BAABEREQQGBlr1l6uhczUSZh4eHowePZqoqCj++Mc/mguHdnV1kZSUhMFgYMeOHWzbtg1FUbj55pu59957LZauPdAcHR2Jj48nPDwcPz8/Xn31VTo6OtDpdIwdOxZnZ2d0Oh0XLlygoqKC0tJS85BIdHQ0aWlppKWlkZKScsPV3cXwtn//fmJiYpgzZw533nnnRY93d3fj7++Pm5ubuRdjGs7v6Oigs7OTrq4uuru76ezsNM8nVVZWUlNTQ21tLeXl5Wg0GkaMGEF4eDhxcXEkJiYyevRonJycbG7uUob7hoFt27bxzjvvcObMGWbNmkVrayu7d+/m/vvvZ8GCBcTFxVm6iYNGURSKi4t56aWXKC0txdfXl1GjRrF9+3bKy8uJiopi3LhxTJo0iRkzZlxU8UGIGxEfH098fDx33nknS5cu7VPGS6VSUVJSwp133sldd93FfffdR1BQEEajkZ6eHg4ePGguCFxYWEhhYSGtra2kpaWRkJBAXFwc8fHxpKSkmPdds2Yy3CfMpk+fTkpKCjk5Obz66quo1Wo++ugjmyoh1J+io6P505/+xAcffMA777xDcXExzz33HGPHjjWn7g/VXqWwrHnz5rF+/Xp+9rOfkZOTw9KlS8nKyjIXlTUl6PT29prnmU0L+J999lmSkpKIi4tj5syZxMfHExUVdVGgG2qfWwlSw0B7ezuurq5kZGSwbNky/v73v+Pu7t5v+z7ZEtP5miaXe3p6+N///V8iIyPNmUzD7T0Rg+eHP/whycnJ7Nu3j9zcXDZv3kxiYiIPPPCAueDBN9nZ2eHt7c1bb72Fk5MTTk5OaDQaNBqNObgN5c+sBKlhoLS0FD8/P/z8/MjMzOSVV14xV1seij0p0/BIYWEhCQkJl9z6oKysjPb2dkJDQ4mJiUGj0QzpP3RhHcLCwpg+fTrR0dHmhd87duzgk08+obe3l7Fjx170MyqVCnt7e6urRzlYJEgNQd3d3XR1dQFfJVH09PRgMBhwcnIiPDwcR0dHysrKCAgIGJJBqqenh5qaGnbs2EF0dPQlg1R+fj5dXV0kJCTg5ORkgVaK4cjOzo6oqCiioqKYPHkylZWV1NbWcujQIcLCwkhJSbF0E62Odc+siatmWqxnGsYqKCjg9OnTKIrCmDFjCAsLQ6VSmcsp5ebmXnb/LlvX2NjI/v37Wb9+fZ99yOC/mVLHjx9Hr9df8purEAPFaDSa55rs7e2JjIzkrrvuwtXVlYaGBtra2izcQusjQWoIyc/Pp729naCgIGJiYqioqLjoGAcHBxYuXMjevXspLy+3QCsHXm1tLTt27GD+/PmX7Cn29PSQm5uLoiiyd48YVJWVlRcFoiNHjtDV1YWHh4esu7sEGe4bQrZt28bkyZOJiYnB09OT+fPnXzTPYmdnx8SJE3n22WeprKyksbERHx8fC7W4/xkMBmpra8nJyeHJJ5+8ZJDat28fXl5eREZGylbpYlAYjUaampq46667AAgODsbPz4/6+nr279/PokWLuPnmmy+ZNq7X69HpdBw6dIjJkycPmWowV0uClA0rKCigs7MTX19fwsLCmDZtGk1NTdTV1eHl5XXZD7OzszOjRo2ioaGBc+fOXTaryBZptVpqamrw8vIiLCzskn/0OTk5hIWFMWLECNnGQgwKlUqFq6srK1eupKKigra2NvR6PV5eXkyYMIHs7Gzi4uKwt7fn+9//Pqmpqbi5uQFfffFqaWnhb3/7G0ajkTFjxpiLRw8HEqRsiKmkfnNzM76+viiKQn19PTU1NQQFBTF69Gjy8/MvmShgYlpHMWbMGKqqqigoKBhSQaqsrIza2lri4uJwcXHp05NUFAWj0cjhw4eZOXOmbGkhBo1KpcLJyYnbbruN8vJyamtrzfsyJSYmmjNte3p6uPXWW/Hw8DDXllSr1Tg6OuLs7MyXX36JnZ0d48aNG1IjIFcic1I2wDTZqigKnZ2dHD16lO7ubuLi4vDx8SEvL4+mpibs7OxITU0lKirqW59z/PjxtLS0cPbsWQwGwyCcxcBTFIXCwkK0Wi0TJky45OOdnZ2cPn2ayMhIIiMjLdBKMZy5urqSmJjItGnTuPXWW5k5cyahoaHmYWlHR0dGjBiBj4+PuYamg4MDwcHB/PSnP+XEiRNs2bKFEydO0NPTY8lTGTQSpGxAZ2cner0etVqNvb09u3btoqyszFyL76677mLXrl3XFGzGjBmDnZ0dFRUVVFdXD2DrB4+iKOTn51NVVcXMmTMvery7u5v9+/cTGhpKaGiolNQSA8705fJGq8+p1Wri4+P57W9/S0NDA7/5zW/MyT82WNnumkiQsgGfffYZu3btoqKiAmdnZ5588kk++ugjCgsLcXBwICQkhAULFlxT9XK1Wk1aWhoBAQFs3bp1AFs/eE6ePElHRwdBQUFERERclDTS0dHB+vXrmTVrFr6+vhZqpRhOenp6aGlpQavV9kswSUpK4rHHHmPixIk88sgjnDp1Cr1e3w8ttV4SpKyQTqejtLSUEydOoCgKkydPpqmpiQMHDtDd3Y2npyeTJ0+mra2N8vJy7OzsrrligkqlYtSoUfj7+5v3WbJ1hw4dwsXFhVGjRl2UEGGqIr1nzx6ysrIkq08MuLa2NtauXcuvfvUrvvzyy355TlPliTlz5rBw4UKeffZZCgsL6ezs7Jfnt0YSpKyEoig0Nzej1+uxs7Oju7ubEydO0NHRQUhICKGhoSiKwtGjR1Gr1SQlJd1wxYioqCj8/f3N+ybZ+rDB8ePHzWP+39TR0UFtbS2tra3ExMTg4uJigRaK4UBRFNrb29m0aROHDx+mtbWVoKCgfnt+Nzc3YmJimDt3Lg4ODnz22WcUFRVdtHB9qJAgZUWKi4tpa2tDo9Hg5uZGXV0dxcXF6PV6UlJSiImJ4eDBg3R1dREQEEBsbCwhISHX/XpBQUEEBgbS1dVFaWmpzSZQmBIizpw5g7u7+yW3HmlsbKSwsJCQkBACAwNxdHS0QEvFUGeqG3nq1Ck+/PBDOjo6mDp1KjNnzuzX2pCenp5kZGRw//33s2nTJvbt20dlZaXN/g1fiQQpK6EoCl9++aU5Uy8oKIgHH3yQv/71rzQ2NuLi4kJ4eDjJycmcOXOmX8ah1Wo1QUFBjBs3js2bN9Pb29sPZzL4FEUhLy8Po9FIUFDQJQN3dXU1R44cYdasWbI2SgwI0065xcXFPPDAA8TGxvLd736XO+64Y0BeT6PRsGjRIm6//XbWrVvHX/7yF1paWoZcMoUEKSuhUql49NFH2b9/PwcOHECv1+Ph4cF3v/tdNm7cSH5+Pv7+/kyfPp20tLR+2+I9LCyMm266ibVr19psSqvBYODzzz8nOzv7sun35eXlHDhw4JoTTIS4WnV1dXzxxRfceuutrFq1iscff5wxY8YM+Os+8sgj3H777dTW1vL973+f9vb2AX/NwSRByopoNBrmzZuHwWBg48aN5rRTJycn2tvb0el02NnZ9eueR76+viQnJ1NZWUl1dbW5erotMRgMbN++nZSUFEJDQy96vKqqCq1Wi4ODA7GxsVa/Y6mwPYWFhfzrX//i7bffZuXKlSxZsgQfH59B+azZ29tz6623smDBApycnHj66adpa2vDaDQO+GsPBvlrHQDd3d3U1taaC0d+s+utKApnz56lpKSEpqYm4L+VIEzJDJ2dneTn5+Ps7ExqaiohISFXrCRxvRwcHPDx8SE6Opq8vDx0Ol2/v8ZA6u3tpb6+Hq1WS3R09CVX4RcVFdHc3HzJKhRC3AjTAvKPP/6Y4uJi0tPTmTdvHoGBgYPaY/fz82Ps2LHMmTOHiooK/vOf/1BXVzckhv0kSA2Ajo4OioqKWLNmDe3t7Zf8RrNz504OHz5MVVVVn/tdXV0JDw8nICCAY8eOYTAYGDVqFOHh4QOSkaZSqdBoNEycOJEjR46Yg6at6Ozs5Ny5c7i7uxMUFHTJ9+js2bPodDrZlkP0K4PBwIULF9i8eTNHjhzB39+fO++8k8TExEH/IqRSqYiIiGDu3LkkJyfz2Wefcfz4cRoaGga1HQNBgtQAaG1t5dixY7z++us0NzdfMkh9+OGHbNq0icLCwosei4iIID4+Hq1WS2dn54B/G9JoNMycOZOcnBwuXLhgU9++mpub2b17NzNmzMDV1fWii4PRaDT3ELOzsy3USjHUGI1GWltb+eKLL3jttdeYP38+3/ve9xg9erTF2qRSqfD19eX555/H0dGRDz/8kB07dthsQpSJBCkrFRoayhNPPIGbm9uAfytzdHRk8uTJ1NbWUlpaalPfvpqamti0aRO33HLLJau+FxUVUV9fj5ubG0lJSRZooRiKCgoKePPNN3n++ed59913uf322wkLC7N0s4Cv/p7/8Y9/4OTkxJo1a/jb3/5m6SbdEAlSVkqlUpkTJAZj6MDOzo6pU6dSVlbGmTNnBvz1+oNOp6O6upqGhgbGjBlzyW3gd+7cSXBwMMnJyTIXJfrF1q1b+fvf/87OnTv517/+RUpKyiV78ZZgaoOLiws/+tGPSEtLY/369bz11ls2m0ghubgDyGg0muecvjmJ2tXVZTUfGpVKhaIoTJgwgUOHDlFUVMTkyZMt3axvVVlZSUlJCaNGjbrsReLgwYPExMQQGxtrFRcRYZtMQ+Dr169nx44dGAwG7r//ftLT03F2draqz5bpi214eDjTp09HpVLx2WefERMTQ0ZGhs1tmihBagAZDAZ27NhxyVTUhoYGqysMmZGRwZYtWygvL6ezs9O8n421Ki8vp7S0lKysrIt6nKYqFOfOnSM7O1u25RDXTVEU9Ho9eXl5bNmyhd7eXjIzM1mwYIFVV9LXaDSMHj0aBwcHjh07xueff46npyfR0dHmDRVtgQSpAdTT08M777yDvb39Rd+0KisrSU1NtVDLLs1UmLWuro7q6mpGjhxp6SZdltFopKysjJKSElatWnXJhImKigp0Ot1lq1AI8W0URaG3t5e6ujpee+01mpqaePTRR5k4caLVf4mDr3bhTkxM5MUXX2Tx4sUEBAQwe/ZskpKSbGZRu2200ka5urqydetWIiIiLvpATJ8+HXd3dwu17GKmObDMzEwqKirYuXOnVQeppqYmcxC6VNZeT08P69atY9KkSYSEhFjVcIywHXq9niNHjrB69Wr0ej1/+ctfCAwMtKkF4RqNhpiYGN577z2+//3vU1VVxX333WczSzJs5522cabhKGu+WKpUKrKysnBzc2Pnzp2Wbs4V7d27F4PBQGZm5iUXOff09LBhwwYmTpxIUFCQVb/vwjq1tbXxpz/9id/+9rf4+/vzj3/8wxygbO3zpFKpSExM5Fe/+hWdnZ288sornDp1yiaWm0iQEn3ExMTg7e1NZWVlv23UNhCOHTuGSqUiPT39ouDf29tLY2MjxcXFjB49Gm9vbwu2VNga0xDfa6+9xunTpxk9ejTf//738fPzs8kABV8FKQcHB1JSUrj55psZMWIEr7zyCpWVlVY3N/5NEqREH56engQEBODq6kp+fr7VZCCaKIpCd3c3586dw97e/pJ7R7W1tXH+/HnzudzInltieFKpVOj1euLj45k+fToZGRlWPxJyNby8vBg7dizZ2dl0dHTwn//8hwsXLlj1gl+ZkxoAarUajUaDl5fXZb95ubu74+LiMiD1+G6EWq0mNDSUxMREdu/ezeTJk61ua4u6ujpqa2tJTU0lOjr6oscbGho4fPgwEyZMwNnZ2abmD4TlqVQq7O3tufPOOwkICMDPz8/STepXoaGhTJ48md7eXn75y18SGxvL2LFj8ff3t8q/Fetr0RDg7u5OWloaP/jBD8yB6ptuv/12Zs+eTUxMjAVaeGWRkZGMGTOGjRs3YjQarWrIT1EUtm/fTnR0NCNHjryol6QoCnV1dezdu5c5c+ZIL0pct1GjRg25AGUSEBDA4sWLefjhh3n++efZtGmT1RaXlp7UAPDw8CAjI4OUlJTL7gB79913mzPqrI2pQkNVVRUlJSWMGDHiktUcLEGlUrF48eLLZkd2dHRQVVXF2bNnmT59uuzAK8RlODg4sGLFChwcHIiOjrbaXX0lSA0AlUqFnZ3dZYfJTJOY1kqtVuPp6cmECRPYu3cvvr6+VhOk4KvUfo1Gc8kAX1RURFVVFampqVZTqkYIa6RSqXB0dGTBggU4OzsPyC4L/cH6vsYLi1OpVLi4uDBhwgQOHDhAa2urpZtkZup9Ojo6XnIxYnFxMbW1tYwdO9ZmM7GEGExBQUF4enpa7RdnCVLikpydncnOzubEiRM0NzdbfZoqfFWGqri4mPr6esaNG2fp5ggh+oEEKXFJTk5OpKen09jYSGVlJS0tLZZu0rdqb2+npKSElpYWxo8fb+nmCCH6gQQpcVl2dnbcfPPNHDx4kNOnT1u6Od/q888/p6enh8TExEHZh0sIMfAkcUJckmnh4ogRI/joo48ICAhg6tSpg/LaBoOB3t5eenp6vvVYRVFQFIX29nbef/99ABISEiRACTFESJASl6VSqZg6dSrvv/8+a9eupaKiYtBf/2oYjUZ0Oh2nTp3i5ptvJj09fYBbJoQYLBKkxBWlpqaSnZ1Nfn4+R44cGZTqE2q1Gnt7+6te42Q0Guno6CApKYnJkyeTnJw8wC0UQgwWCVLiskzruV555RUURcHOzs4m9tARQgwdEqTEt7LWRX5CiKFPgpT4VpKEIISwFElBF0IIYbUkSAkhhLBa1xSkXn75ZcaNG4e7uzsBAQEsWrSIgoKCPsdMmzatz1bpKpWKhx9+uM8x5eXl3HLLLbi4uBAQEMCTTz5pE2V3hBBCDK5rmpPatWsXK1euZNy4cej1en7yk58we/Zs8vPzcXV1NR+3YsUKfv7zn5v//fWJd4PBwC233EJQUBD79++npqaG+++/HwcHB/7v//6vH05JCCHEUKFSbmBHu/r6egICAti1axdTpkwBvupJpaWl8dprr13yZzZu3Mitt95KdXU1gYGBAPz5z3/m6aefpr6+/qrWxuh0Ojw9PWlpacHDw+N6my+EEMJCrvY6fkNzUqaioz4+Pn3u//e//42fnx+jR4/mmWeeoaOjw/xYTk4OycnJ5gAFMGfOHHQ6HXl5eZd8ne7ubnQ6XZ+bEEKIoe+6U9CNRiOPP/44EydOZPTo0eb777nnHiIjIwkJCeHkyZM8/fTTFBQUsGbNGgC0Wm2fAAWY/63Vai/5Wi+//DIvvvji9TZVCCGEjbruILVy5UpOnz7N3r17+9z/0EMPmf8/OTmZ4OBgZsyYwfnz5xk5cuR1vdYzzzzDE088Yf63TqcjPDz8+houhBDCZlzXcN+qVatYv349O3bsICws7IrHZmVlAV9t6w1f7QJZW1vb5xjTv4OCgi75HBqNBg8Pjz43IYQQQ981BSlFUVi1ahVr165l+/btREVFfevPHD9+HIDg4GAAsrOzOXXqFHV1deZjtm7dioeHB6NGjbqW5gghhBjirmm4b+XKlbz33nt89tlnuLu7m+eQPD09cXZ25vz587z33nvcfPPN+Pr6cvLkSX74wx8yZcoUUlJSAJg9ezajRo3iO9/5Dr/+9a/RarU899xzrFy5Eo1G0/9nKIQQwmZdUwr65Wq4vfXWWyxfvpyKigruu+8+Tp8+TXt7O+Hh4dx2220899xzfYboysrKeOSRR9i5cyeurq4sW7aMV155BXv7q4uZkoIuhBC27Wqv4ze0TspSJEgJIYRtG5R1UkIIIcRAkiAlhBDCakmQEkIIYbUkSAkhhLBaEqSEEEJYLQlSQgghrJYEKSGEEFZLgpQQQgirJUFKCCGE1ZIgJYQQwmpJkBJCCGG1JEgJIYSwWhKkhBBCWC0JUkIIIayWBCkhhBBWS4KUEEIIqyVBSgghhNWSICWEEMJqSZASQghhtSRICSGEsFoSpIQQQlgtCVJCCCGslgQpIYQQVkuClBBCCKslQUoIIYTVkiAlhBDCakmQEkIIYbUkSAkhhLBaEqSEEEJYLQlSQgghrJYEKSGEEFZLgpQQQgirJUFKCCGE1ZIgJYQQwmpJkBJCCGG1JEgJIYSwWhKkhBBCWC0JUkIIIayWBCkhhBBWS4KUEEIIqyVBSgghhNWSICWEEMJqSZASQghhtSRICSGEsFoSpIQQQlgtCVJCCCGslgQpIYQQVkuClBBCCKslQUoIIYTVkiAlhBDCakmQEkIIYbUkSAkhhLBaEqSEEEJYLXtLN+B6KIoCgE6ns3BLhBBCXA/T9dt0Pb8cmwxSra2tAISHh1u4JUIIIW5Ea2srnp6el31cpXxbGLNCRqORgoICRo0aRUVFBR4eHpZu0qDQ6XSEh4cPq3MGOe/hdN7D8ZxheJ63oii0trYSEhKCWn35mSeb7Emp1WpCQ0MB8PDwGDa/VJPheM4g5z2cDMdzhuF33lfqQZlI4oQQQgirJUFKCCGE1bLZIKXRaHjhhRfQaDSWbsqgGY7nDHLew+m8h+M5w/A976thk4kTQgghhgeb7UkJIYQY+iRICSGEsFoSpIQQQlgtCVJCCCGslk0GqTfeeIMRI0bg5OREVlYWhw4dsnST+tXPfvYzVCpVn1tCQoL58a6uLlauXImvry9ubm4sWbKE2tpaC7b42u3evZv58+cTEhKCSqXi008/7fO4oig8//zzBAcH4+zszMyZMyksLOxzTGNjI/feey8eHh54eXnx4IMP0tbWNohnce2+7byXL19+0e9+7ty5fY6xtfN++eWXGTduHO7u7gQEBLBo0SIKCgr6HHM1n+ny8nJuueUWXFxcCAgI4Mknn0Sv1w/mqVyTqznvadOmXfT7fvjhh/scY2vn3d9sLkh9+OGHPPHEE7zwwgscPXqU1NRU5syZQ11dnaWb1q+SkpKoqakx3/bu3Wt+7Ic//CHr1q3j448/ZteuXVRXV7N48WILtvbatbe3k5qayhtvvHHJx3/961/zhz/8gT//+c8cPHgQV1dX5syZQ1dXl/mYe++9l7y8PLZu3cr69evZvXs3Dz300GCdwnX5tvMGmDt3bp/f/fvvv9/ncVs77127drFy5UoOHDjA1q1b6e3tZfbs2bS3t5uP+bbPtMFg4JZbbqGnp4f9+/fzzjvv8Pbbb/P8889b4pSuytWcN8CKFSv6/L5//etfmx+zxfPud4qNyczMVFauXGn+t8FgUEJCQpSXX37Zgq3qXy+88IKSmpp6yceam5sVBwcH5eOPPzbfd+bMGQVQcnJyBqmF/QtQ1q5da/630WhUgoKClNWrV5vva25uVjQajfL+++8riqIo+fn5CqAcPnzYfMzGjRsVlUqlVFVVDVrbb8Q3z1tRFGXZsmXKwoULL/szQ+G86+rqFEDZtWuXoihX95nesGGDolarFa1Waz7mT3/6k+Lh4aF0d3cP7glcp2+et6IoytSpU5XHHnvssj8zFM77RtlUT6qnp4fc3Fxmzpxpvk+tVjNz5kxycnIs2LL+V1hYSEhICNHR0dx7772Ul5cDkJubS29vb5/3ICEhgYiIiCHzHpSUlKDVavuco6enJ1lZWeZzzMnJwcvLi7Fjx5qPmTlzJmq1moMHDw56m/vTzp07CQgIID4+nkceeYSGhgbzY0PhvFtaWgDw8fEBru4znZOTQ3JyMoGBgeZj5syZg06nIy8vbxBbf/2+ed4m//73v/Hz82P06NE888wzdHR0mB8bCud9o2yqwOyFCxcwGAx9fmEAgYGBnD171kKt6n9ZWVm8/fbbxMfHU1NTw4svvsjkyZM5ffo0Wq0WR0dHvLy8+vxMYGAgWq3WMg3uZ6bzuNTv2fSYVqslICCgz+P29vb4+PjY9Pswd+5cFi9eTFRUFOfPn+cnP/kJ8+bNIycnBzs7O5s/b6PRyOOPP87EiRMZPXo0wFV9prVa7SU/D6bHrN2lzhvgnnvuITIykpCQEE6ePMnTTz9NQUEBa9asAWz/vPuDTQWp4WLevHnm/09JSSErK4vIyEg++ugjnJ2dLdgyMdCWLl1q/v/k5GRSUlIYOXIkO3fuZMaMGRZsWf9YuXIlp0+f7jPHOhxc7ry/PpeYnJxMcHAwM2bM4Pz584wcOXKwm2mVbGq4z8/PDzs7u4uyfmprawkKCrJQqwael5cXcXFxFBUVERQURE9PD83NzX2OGUrvgek8rvR7DgoKuihZRq/X09jYOGTeB4Do6Gj8/PwoKioCbPu8V61axfr169mxYwdhYWHm+6/mMx0UFHTJz4PpMWt2ufO+lKysLIA+v29bPe/+YlNBytHRkTFjxvDll1+a7zMajXz55ZdkZ2dbsGUDq62tjfPnzxMcHMyYMWNwcHDo8x4UFBRQXl4+ZN6DqKgogoKC+pyjTqfj4MGD5nPMzs6mubmZ3Nxc8zHbt2/HaDSa/9CHgsrKShoaGggODgZs87wVRWHVqlWsXbuW7du3ExUV1efxq/lMZ2dnc+rUqT4BeuvWrXh4eDBq1KjBOZFr9G3nfSnHjx8H6PP7trXz7neWzty4Vh988IGi0WiUt99+W8nPz1ceeughxcvLq0/2i6370Y9+pOzcuVMpKSlR9u3bp8ycOVPx8/NT6urqFEVRlIcffliJiIhQtm/frhw5ckTJzs5WsrOzLdzqa9Pa2qocO3ZMOXbsmAIor776qnLs2DGlrKxMURRFeeWVVxQvLy/ls88+U06ePKksXLhQiYqKUjo7O83PMXfuXCU9PV05ePCgsnfvXiU2Nla5++67LXVKV+VK593a2qr8+Mc/VnJycpSSkhJl27ZtSkZGhhIbG6t0dXWZn8PWzvuRRx5RPD09lZ07dyo1NTXmW0dHh/mYb/tM6/V6ZfTo0crs2bOV48ePK5s2bVL8/f2VZ555xhKndFW+7byLioqUn//858qRI0eUkpIS5bPPPlOio6OVKVOmmJ/DFs+7v9lckFIURXn99deViIgIxdHRUcnMzFQOHDhg6Sb1q7vuuksJDg5WHB0dldDQUOWuu+5SioqKzI93dnYqP/jBDxRvb2/FxcVFue2225SamhoLtvja7dixQwEuui1btkxRlK/S0H/6058qgYGBikajUWbMmKEUFBT0eY6Ghgbl7rvvVtzc3BQPDw/lgQceUFpbWy1wNlfvSufd0dGhzJ49W/H391ccHByUyMhIZcWKFRd9AbO1877U+QLKW2+9ZT7maj7TpaWlyrx58xRnZ2fFz89P+dGPfqT09vYO8tlcvW877/LycmXKlCmKj4+PotFolJiYGOXJJ59UWlpa+jyPrZ13f5OtOoQQQlgtm5qTEkIIMbxIkBJCCGG1JEgJIYSwWhKkhBBCWC0JUkIIIayWBCkhhBBWS4KUEEIIqyVBSgghhNWSICWEEMJqSZASQghhtSRICSGEsFoSpIQQQlit/w+53jACNBP/vwAAAABJRU5ErkJggg==\n",
            "text/plain": [
              "<Figure size 640x480 with 1 Axes>"
            ]
          },
          "metadata": {},
          "output_type": "display_data"
        }
      ],
      "source": [
        "#Demo of extra image.\n",
        "image = cv2.imread(extra['file_path'][1]).astype(np.uint8)\n",
        "plt.imshow(image)"
      ],
      "id": "7e9da4a3-8439-429b-9fc2-ca195ed950a9"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "4d8ac6b7-3502-499b-8867-4a4fa5d4f621"
      },
      "outputs": [],
      "source": [
        "#Concat train and extra dataset.\n",
        "# train = pd.read_pickle('./bms-molecular-translation/train1.pkl')\n",
        "# extra = pd.read_pickle('./bms-molecular-translation/extra1.pkl')\n",
        "# tokenizer = torch.load('./bms-molecular-translation/tokenizer.pth')\n",
        "full  = pd.concat([train[['InChI_text', 'file_path', 'text_length']], extra[['InChI_text', 'file_path', 'text_length']]], axis = 0).reset_index(drop = True)\n",
        "#max_size = shape_after_crop(full)"
      ],
      "id": "4d8ac6b7-3502-499b-8867-4a4fa5d4f621"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FcePOINZTQCM",
        "outputId": "ad6d1a49-58b1-42a4-e427-8b0e91a26e78"
      },
      "outputs": [
        {
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "100%|██████████| 200002/200002 [08:24<00:00, 396.67it/s]\n"
          ]
        }
      ],
      "source": [
        "#Preprocess the images in full dataset. Including resize, add noise, and randomly selected transformation from vertical flip and rotate. The whole process will double the amount of full dataset.\n",
        "#Each original image will have a flipped or rotated copy that has the identical caption, the InChI text.\n",
        "full_df = preprocess_train_images(full, get_transform, CFG)"
      ],
      "id": "FcePOINZTQCM"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "N2ZoJbILTZqj",
        "outputId": "82b7e657-9156-4d8e-b36b-b6388918d7ab"
      },
      "outputs": [
        {
          "data": {
            "text/html": [
              "\n",
              "  <div id=\"df-2e5970ab-b19b-463d-b700-99f3659c0dd2\" class=\"colab-df-container\">\n",
              "    <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>InChI_text</th>\n",
              "      <th>file_path</th>\n",
              "      <th>text_length</th>\n",
              "      <th>image</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>C 21 H 25 N O 5 /c 1 - 15 - 5 - 7 - 16 ( 8 - 6...</td>\n",
              "      <td>./bms-molecular-translation/train/0/4/4/044921...</td>\n",
              "      <td>82</td>\n",
              "      <td>[[[0], [0], [255], [0], [0], [0], [0], [0], [2...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>C 12 H 23 N 3 O /c 1 - 9 ( 2 ) 11 - 4 - 3 - 7 ...</td>\n",
              "      <td>./bms-molecular-translation/train/0/1/d/01d620...</td>\n",
              "      <td>64</td>\n",
              "      <td>[[[0], [0], [0], [0], [0], [0], [0], [0], [0],...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>C 16 H 25 Br N 2 /c 1 - 18 - 11 - 14 - 8 - 9 -...</td>\n",
              "      <td>./bms-molecular-translation/train_extra/extra_...</td>\n",
              "      <td>74</td>\n",
              "      <td>[[[0], [0], [0], [0], [255], [255], [255], [0]...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>C 19 H 19 N 3 O 5 /c 1 - 12 - 3 - 5 - 14 ( 6 -...</td>\n",
              "      <td>./bms-molecular-translation/train_extra/extra_...</td>\n",
              "      <td>109</td>\n",
              "      <td>[[[0], [0], [0], [0], [0], [0], [0], [0], [0],...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>C 17 H 18 N 4 O S /c 18 - 16 ( 22 ) 15 - 5 - 1...</td>\n",
              "      <td>./bms-molecular-translation/train/0/0/3/003631...</td>\n",
              "      <td>88</td>\n",
              "      <td>[[[255], [0], [0], [255], [0], [0], [0], [0], ...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "    <div class=\"colab-df-buttons\">\n",
              "\n",
              "  <div class=\"colab-df-container\">\n",
              "    <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-2e5970ab-b19b-463d-b700-99f3659c0dd2')\"\n",
              "            title=\"Convert this dataframe to an interactive table.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\" viewBox=\"0 -960 960 960\">\n",
              "    <path d=\"M120-120v-720h720v720H120Zm60-500h600v-160H180v160Zm220 220h160v-160H400v160Zm0 220h160v-160H400v160ZM180-400h160v-160H180v160Zm440 0h160v-160H620v160ZM180-180h160v-160H180v160Zm440 0h160v-160H620v160Z\"/>\n",
              "  </svg>\n",
              "    </button>\n",
              "\n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    .colab-df-buttons div {\n",
              "      margin-bottom: 4px;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "    <script>\n",
              "      const buttonEl =\n",
              "        document.querySelector('#df-2e5970ab-b19b-463d-b700-99f3659c0dd2 button.colab-df-convert');\n",
              "      buttonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "      async function convertToInteractive(key) {\n",
              "        const element = document.querySelector('#df-2e5970ab-b19b-463d-b700-99f3659c0dd2');\n",
              "        const dataTable =\n",
              "          await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                    [key], {});\n",
              "        if (!dataTable) return;\n",
              "\n",
              "        const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "          '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "          + ' to learn more about interactive tables.';\n",
              "        element.innerHTML = '';\n",
              "        dataTable['output_type'] = 'display_data';\n",
              "        await google.colab.output.renderOutput(dataTable, element);\n",
              "        const docLink = document.createElement('div');\n",
              "        docLink.innerHTML = docLinkHtml;\n",
              "        element.appendChild(docLink);\n",
              "      }\n",
              "    </script>\n",
              "  </div>\n",
              "\n",
              "\n",
              "<div id=\"df-1c131b43-1ecb-4e7e-93ce-0c82935b14fe\">\n",
              "  <button class=\"colab-df-quickchart\" onclick=\"quickchart('df-1c131b43-1ecb-4e7e-93ce-0c82935b14fe')\"\n",
              "            title=\"Suggest charts.\"\n",
              "            style=\"display:none;\">\n",
              "\n",
              "<svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "     width=\"24px\">\n",
              "    <g>\n",
              "        <path d=\"M19 3H5c-1.1 0-2 .9-2 2v14c0 1.1.9 2 2 2h14c1.1 0 2-.9 2-2V5c0-1.1-.9-2-2-2zM9 17H7v-7h2v7zm4 0h-2V7h2v10zm4 0h-2v-4h2v4z\"/>\n",
              "    </g>\n",
              "</svg>\n",
              "  </button>\n",
              "\n",
              "<style>\n",
              "  .colab-df-quickchart {\n",
              "      --bg-color: #E8F0FE;\n",
              "      --fill-color: #1967D2;\n",
              "      --hover-bg-color: #E2EBFA;\n",
              "      --hover-fill-color: #174EA6;\n",
              "      --disabled-fill-color: #AAA;\n",
              "      --disabled-bg-color: #DDD;\n",
              "  }\n",
              "\n",
              "  [theme=dark] .colab-df-quickchart {\n",
              "      --bg-color: #3B4455;\n",
              "      --fill-color: #D2E3FC;\n",
              "      --hover-bg-color: #434B5C;\n",
              "      --hover-fill-color: #FFFFFF;\n",
              "      --disabled-bg-color: #3B4455;\n",
              "      --disabled-fill-color: #666;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart {\n",
              "    background-color: var(--bg-color);\n",
              "    border: none;\n",
              "    border-radius: 50%;\n",
              "    cursor: pointer;\n",
              "    display: none;\n",
              "    fill: var(--fill-color);\n",
              "    height: 32px;\n",
              "    padding: 0;\n",
              "    width: 32px;\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart:hover {\n",
              "    background-color: var(--hover-bg-color);\n",
              "    box-shadow: 0 1px 2px rgba(60, 64, 67, 0.3), 0 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "    fill: var(--button-hover-fill-color);\n",
              "  }\n",
              "\n",
              "  .colab-df-quickchart-complete:disabled,\n",
              "  .colab-df-quickchart-complete:disabled:hover {\n",
              "    background-color: var(--disabled-bg-color);\n",
              "    fill: var(--disabled-fill-color);\n",
              "    box-shadow: none;\n",
              "  }\n",
              "\n",
              "  .colab-df-spinner {\n",
              "    border: 2px solid var(--fill-color);\n",
              "    border-color: transparent;\n",
              "    border-bottom-color: var(--fill-color);\n",
              "    animation:\n",
              "      spin 1s steps(1) infinite;\n",
              "  }\n",
              "\n",
              "  @keyframes spin {\n",
              "    0% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "      border-left-color: var(--fill-color);\n",
              "    }\n",
              "    20% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    30% {\n",
              "      border-color: transparent;\n",
              "      border-left-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    40% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-top-color: var(--fill-color);\n",
              "    }\n",
              "    60% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "    }\n",
              "    80% {\n",
              "      border-color: transparent;\n",
              "      border-right-color: var(--fill-color);\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "    90% {\n",
              "      border-color: transparent;\n",
              "      border-bottom-color: var(--fill-color);\n",
              "    }\n",
              "  }\n",
              "</style>\n",
              "\n",
              "  <script>\n",
              "    async function quickchart(key) {\n",
              "      const quickchartButtonEl =\n",
              "        document.querySelector('#' + key + ' button');\n",
              "      quickchartButtonEl.disabled = true;  // To prevent multiple clicks.\n",
              "      quickchartButtonEl.classList.add('colab-df-spinner');\n",
              "      try {\n",
              "        const charts = await google.colab.kernel.invokeFunction(\n",
              "            'suggestCharts', [key], {});\n",
              "      } catch (error) {\n",
              "        console.error('Error during call to suggestCharts:', error);\n",
              "      }\n",
              "      quickchartButtonEl.classList.remove('colab-df-spinner');\n",
              "      quickchartButtonEl.classList.add('colab-df-quickchart-complete');\n",
              "    }\n",
              "    (() => {\n",
              "      let quickchartButtonEl =\n",
              "        document.querySelector('#df-1c131b43-1ecb-4e7e-93ce-0c82935b14fe button');\n",
              "      quickchartButtonEl.style.display =\n",
              "        google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "    })();\n",
              "  </script>\n",
              "</div>\n",
              "\n",
              "    </div>\n",
              "  </div>\n"
            ],
            "text/plain": [
              "                                          InChI_text  \\\n",
              "0  C 21 H 25 N O 5 /c 1 - 15 - 5 - 7 - 16 ( 8 - 6...   \n",
              "1  C 12 H 23 N 3 O /c 1 - 9 ( 2 ) 11 - 4 - 3 - 7 ...   \n",
              "2  C 16 H 25 Br N 2 /c 1 - 18 - 11 - 14 - 8 - 9 -...   \n",
              "3  C 19 H 19 N 3 O 5 /c 1 - 12 - 3 - 5 - 14 ( 6 -...   \n",
              "4  C 17 H 18 N 4 O S /c 18 - 16 ( 22 ) 15 - 5 - 1...   \n",
              "\n",
              "                                           file_path  text_length  \\\n",
              "0  ./bms-molecular-translation/train/0/4/4/044921...           82   \n",
              "1  ./bms-molecular-translation/train/0/1/d/01d620...           64   \n",
              "2  ./bms-molecular-translation/train_extra/extra_...           74   \n",
              "3  ./bms-molecular-translation/train_extra/extra_...          109   \n",
              "4  ./bms-molecular-translation/train/0/0/3/003631...           88   \n",
              "\n",
              "                                               image  \n",
              "0  [[[0], [0], [255], [0], [0], [0], [0], [0], [2...  \n",
              "1  [[[0], [0], [0], [0], [0], [0], [0], [0], [0],...  \n",
              "2  [[[0], [0], [0], [0], [255], [255], [255], [0]...  \n",
              "3  [[[0], [0], [0], [0], [0], [0], [0], [0], [0],...  \n",
              "4  [[[255], [0], [0], [255], [0], [0], [0], [0], ...  "
            ]
          },
          "execution_count": 28,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Optional, check the columns' names of extra and train dataset.\n",
        "# full_df.head()"
      ],
      "id": "N2ZoJbILTZqj"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "f475f55a-cebe-4076-8d7c-c9d0dd685770"
      },
      "outputs": [],
      "source": [
        "#Optional. Save the full_df ds as .pkl files.\n",
        "full_df.to_pickle('./bms-molecular-translation/fulldf1.pkl')\n",
        "# full_df = pd.read_pickle('./bms-molecular-translation/fulldf1.pkl')"
      ],
      "id": "f475f55a-cebe-4076-8d7c-c9d0dd685770"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "b401a292-51da-4a41-84fd-c9d5448d0ede",
        "outputId": "25527132-9c40-4b29-ee59-de3fa66de3f1"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "5     26667\n",
              "11    26667\n",
              "12    26667\n",
              "1     26667\n",
              "3     26667\n",
              "10    26667\n",
              "6     26667\n",
              "9     26667\n",
              "2     26667\n",
              "8     26667\n",
              "0     26667\n",
              "13    26667\n",
              "4     26667\n",
              "7     26667\n",
              "14    26666\n",
              "Name: fold, dtype: int64"
            ]
          },
          "execution_count": 30,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Prepare for the cross validation. Create the column fold to show which ones will be used for validation in individual epoch.\n",
        "kf = KFold(n_splits = CFG.num_folds, shuffle = True, random_state = CFG.seed)\n",
        "for i, (train_idx, val_idx) in enumerate(kf.split(full_df)):\n",
        "    full_df.loc[val_idx, 'fold'] = i\n",
        "full_df['fold'] = full_df['fold'].astype(int)\n",
        "full_df['fold'].value_counts()"
      ],
      "id": "b401a292-51da-4a41-84fd-c9d5448d0ede"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "7871522d-a577-4b0c-a9ce-f8194cc7823d",
        "outputId": "e32697e7-4c86-4c08-842b-53e28426c53a"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "268"
            ]
          },
          "execution_count": 31,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Check max_text_len, if this NO. changes, go back to class CFG and modify it.\n",
        "max_text_len = full_df['text_length'].max()\n",
        "text_len = max_text_len + 2\n",
        "text_len"
      ],
      "id": "7871522d-a577-4b0c-a9ce-f8194cc7823d"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "vIqEYnGmGC3t",
        "outputId": "16e4bdd1-a1a3-4219-e9c4-f5f245534013"
      },
      "outputs": [
        {
          "data": {
            "text/plain": [
              "175"
            ]
          },
          "execution_count": 32,
          "metadata": {},
          "output_type": "execute_result"
        }
      ],
      "source": [
        "#Check vocab length, if this NO. changes, go back to class CFG and modify it.\n",
        "len(tokenizer.text2index)"
      ],
      "id": "vIqEYnGmGC3t"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "colab": {
          "background_save": true,
          "base_uri": "https://localhost:8080/"
        },
        "id": "714c8627-2309-44c0-aec5-9e9df6b91e37",
        "outputId": "67792748-95b3-442e-b51c-d3c839b0bd61"
      },
      "outputs": [
        {
          "metadata": {
            "tags": null
          },
          "name": "stdout",
          "output_type": "stream",
          "text": [
            "Train worker process using GPU: 0 for training\n"
          ]
        },
        {
          "metadata": {
            "tags": null
          },
          "name": "stderr",
          "output_type": "stream",
          "text": [
            "Downloading: \"https://download.pytorch.org/models/resnet18-f37072fd.pth\" to /root/.cache/torch/hub/checkpoints/resnet18-f37072fd.pth\n",
            "100%|██████████| 44.7M/44.7M [00:00<00:00, 141MB/s]\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Now Training Epoch 0 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   5.58 | Tokens / Sec:  3175.3 | Learning Rate: 2.2e-05\n",
            "Batch    101 in the current epoch | Loss / Token:   2.20 | Tokens / Sec:  8892.2 | Learning Rate: 1.1e-03\n",
            "Batch    201 in the current epoch | Loss / Token:   1.90 | Tokens / Sec:  8219.9 | Learning Rate: 2.2e-03\n",
            "Batch    301 in the current epoch | Loss / Token:   1.69 | Tokens / Sec:  8042.5 | Learning Rate: 3.3e-03\n",
            "Batch    401 in the current epoch | Loss / Token:   1.65 | Tokens / Sec:  7996.2 | Learning Rate: 4.4e-03\n",
            "Batch    501 in the current epoch | Loss / Token:   1.51 | Tokens / Sec:  7989.9 | Learning Rate: 3.9e-03\n",
            "Batch    601 in the current epoch | Loss / Token:   1.49 | Tokens / Sec:  8018.8 | Learning Rate: 3.6e-03\n",
            "Batch    701 in the current epoch | Loss / Token:   1.43 | Tokens / Sec:  8039.2 | Learning Rate: 3.3e-03\n",
            "Batch    801 in the current epoch | Loss / Token:   1.37 | Tokens / Sec:  7992.3 | Learning Rate: 3.1e-03\n",
            "Batch    901 in the current epoch | Loss / Token:   1.43 | Tokens / Sec:  8003.7 | Learning Rate: 2.9e-03\n",
            "Batch   1001 in the current epoch | Loss / Token:   1.48 | Tokens / Sec:  8017.5 | Learning Rate: 2.8e-03\n",
            "Batch   1101 in the current epoch | Loss / Token:   1.33 | Tokens / Sec:  7967.2 | Learning Rate: 2.7e-03\n",
            "Batch   1201 in the current epoch | Loss / Token:   1.33 | Tokens / Sec:  8014.5 | Learning Rate: 2.5e-03\n",
            "Batch   1301 in the current epoch | Loss / Token:   1.30 | Tokens / Sec:  7976.0 | Learning Rate: 2.4e-03\n",
            "Batch   1401 in the current epoch | Loss / Token:   1.29 | Tokens / Sec:  8015.9 | Learning Rate: 2.4e-03\n",
            "Batch   1501 in the current epoch | Loss / Token:   1.32 | Tokens / Sec:  7990.9 | Learning Rate: 2.3e-03\n",
            "Batch   1601 in the current epoch | Loss / Token:   1.28 | Tokens / Sec:  8020.2 | Learning Rate: 2.2e-03\n",
            "Batch   1701 in the current epoch | Loss / Token:   1.30 | Tokens / Sec:  8029.0 | Learning Rate: 2.1e-03\n",
            "Batch   1801 in the current epoch | Loss / Token:   1.27 | Tokens / Sec:  8045.7 | Learning Rate: 2.1e-03\n",
            "Batch   1901 in the current epoch | Loss / Token:   1.23 | Tokens / Sec:  8019.7 | Learning Rate: 2.0e-03\n",
            "Batch   2001 in the current epoch | Loss / Token:   1.23 | Tokens / Sec:  7982.3 | Learning Rate: 2.0e-03\n",
            "Batch   2101 in the current epoch | Loss / Token:   1.21 | Tokens / Sec:  7954.3 | Learning Rate: 1.9e-03\n",
            "Batch   2201 in the current epoch | Loss / Token:   1.25 | Tokens / Sec:  8041.8 | Learning Rate: 1.9e-03\n",
            "Batch   2301 in the current epoch | Loss / Token:   1.19 | Tokens / Sec:  7998.9 | Learning Rate: 1.8e-03\n",
            "Batch   2401 in the current epoch | Loss / Token:   1.23 | Tokens / Sec:  8018.2 | Learning Rate: 1.8e-03\n",
            "Batch   2501 in the current epoch | Loss / Token:   1.27 | Tokens / Sec:  8024.2 | Learning Rate: 1.8e-03\n",
            "Batch   2601 in the current epoch | Loss / Token:   1.27 | Tokens / Sec:  8000.8 | Learning Rate: 1.7e-03\n",
            "Batch   2701 in the current epoch | Loss / Token:   1.20 | Tokens / Sec:  8003.8 | Learning Rate: 1.7e-03\n",
            "Batch   2801 in the current epoch | Loss / Token:   1.14 | Tokens / Sec:  7992.7 | Learning Rate: 1.7e-03\n",
            "Batch   2901 in the current epoch | Loss / Token:   1.21 | Tokens / Sec:  8017.6 | Learning Rate: 1.6e-03\n",
            "Batch   3001 in the current epoch | Loss / Token:   1.18 | Tokens / Sec:  8010.9 | Learning Rate: 1.6e-03\n",
            "Batch   3101 in the current epoch | Loss / Token:   1.21 | Tokens / Sec:  8060.0 | Learning Rate: 1.6e-03\n",
            "Batch   3201 in the current epoch | Loss / Token:   1.13 | Tokens / Sec:  8046.4 | Learning Rate: 1.6e-03\n",
            "Batch   3301 in the current epoch | Loss / Token:   1.15 | Tokens / Sec:  8003.0 | Learning Rate: 1.5e-03\n",
            "Batch   3401 in the current epoch | Loss / Token:   1.14 | Tokens / Sec:  8037.8 | Learning Rate: 1.5e-03\n",
            "Batch   3501 in the current epoch | Loss / Token:   1.14 | Tokens / Sec:  8026.0 | Learning Rate: 1.5e-03\n",
            "Batch   3601 in the current epoch | Loss / Token:   1.13 | Tokens / Sec:  8015.0 | Learning Rate: 1.5e-03\n",
            "Batch   3701 in the current epoch | Loss / Token:   1.12 | Tokens / Sec:  7981.9 | Learning Rate: 1.5e-03\n",
            "Batch   3801 in the current epoch | Loss / Token:   1.09 | Tokens / Sec:  8019.1 | Learning Rate: 1.4e-03\n",
            "Batch   3901 in the current epoch | Loss / Token:   1.09 | Tokens / Sec:  8022.4 | Learning Rate: 1.4e-03\n",
            "Batch   4001 in the current epoch | Loss / Token:   1.18 | Tokens / Sec:  8017.6 | Learning Rate: 1.4e-03\n",
            "Batch   4101 in the current epoch | Loss / Token:   1.12 | Tokens / Sec:  7974.8 | Learning Rate: 1.4e-03\n",
            "Batch   4201 in the current epoch | Loss / Token:   1.11 | Tokens / Sec:  8006.9 | Learning Rate: 1.4e-03\n",
            "Batch   4301 in the current epoch | Loss / Token:   1.15 | Tokens / Sec:  8008.9 | Learning Rate: 1.3e-03\n",
            "Batch   4401 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  8058.3 | Learning Rate: 1.3e-03\n",
            "Batch   4501 in the current epoch | Loss / Token:   1.10 | Tokens / Sec:  7989.9 | Learning Rate: 1.3e-03\n",
            "Batch   4601 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  8012.1 | Learning Rate: 1.3e-03\n",
            "Batch   4701 in the current epoch | Loss / Token:   1.05 | Tokens / Sec:  7980.9 | Learning Rate: 1.3e-03\n",
            "Batch   4801 in the current epoch | Loss / Token:   1.12 | Tokens / Sec:  7979.2 | Learning Rate: 1.3e-03\n",
            "Batch   4901 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  7983.9 | Learning Rate: 1.3e-03\n",
            "Batch   5001 in the current epoch | Loss / Token:   1.11 | Tokens / Sec:  8073.6 | Learning Rate: 1.2e-03\n",
            "Batch   5101 in the current epoch | Loss / Token:   1.06 | Tokens / Sec:  7971.8 | Learning Rate: 1.2e-03\n",
            "Batch   5201 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  8062.0 | Learning Rate: 1.2e-03\n",
            "Batch   5301 in the current epoch | Loss / Token:   1.07 | Tokens / Sec:  8017.6 | Learning Rate: 1.2e-03\n",
            "Batch   5401 in the current epoch | Loss / Token:   1.06 | Tokens / Sec:  7986.7 | Learning Rate: 1.2e-03\n",
            "Batch   5501 in the current epoch | Loss / Token:   1.09 | Tokens / Sec:  7979.9 | Learning Rate: 1.2e-03\n",
            "Batch   5601 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  8035.7 | Learning Rate: 1.2e-03\n",
            "Batch   5701 in the current epoch | Loss / Token:   1.12 | Tokens / Sec:  8010.5 | Learning Rate: 1.2e-03\n",
            "Batch   5801 in the current epoch | Loss / Token:   1.11 | Tokens / Sec:  7985.1 | Learning Rate: 1.2e-03\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 94% | 53% |\n",
            "Now Validating Epoch 0 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  31.41\n",
            "Randomly sampled reference InChI is ['C', '9', 'H', '15', 'F', '6', 'N', 'O', '/c', '1', '-', '4', '-', '16', '-', '5', '(', '2', ')', '6', '(', '3', ')', '17', '-', '7', '(', '8', '(', '10', ',', '11', ')', '12', ')', '9', '(', '13', ',', '14', ')', '15', '/h', '5', '-', '7', ',', '16', 'H', ',', '4', 'H', '2', ',', '1', '-', '3', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '9', 'H', '13', 'F', '3', 'N', 'O', '2', '/c', '1', '-', '4', '(', '2', ')', '7', '(', '12', ')', '5', '(', '3', ')', '6', '(', '11', ')', '9', '(', '12', ')', '11', '(', '12', ')', '12', '/h', '4', '-', '5', 'H', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '12', ',', '14', ')', '(', 'H', ',', '13', ',', '14', ')']\n",
            "Batch    101 in the current validation dataloader | Bleu score  32.72\n",
            "Randomly sampled reference InChI is ['C', '37', 'H', '69', 'O', '8', 'P', '/c', '1', '-', '3', '-', '5', '-', '7', '-', '9', '-', '11', '-', '13', '-', '15', '-', '17', '-', '18', '-', '20', '-', '22', '-', '24', '-', '26', '-', '28', '-', '30', '-', '32', '-', '37', '(', '39', ')', '45', '-', '35', '(', '34', '-', '44', '-', '46', '(', '40', ',', '41', ')', '42', ')', '33', '-', '43', '-', '36', '(', '38', ')', '31', '-', '29', '-', '27', '-', '25', '-', '23', '-', '21', '-', '19', '-', '16', '-', '14', '-', '12', '-', '10', '-', '8', '-', '6', '-', '4', '-', '2', '/h', '14', ',', '16', '-', '18', ',', '35', 'H', ',', '3', '-', '13', ',', '15', ',', '19', '-', '34', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', '2', ',', '40', ',', '41', ',', '42', ')', '/b', '16', '-', '14', '-', ',', '18', '-', '17', '-']\n",
            "the corresponding InChI predicted is ['C', '45', 'H', '98', 'O', '7', '/c', '1', '-', '3', '-', '5', '-', '7', '-', '9', '-', '11', '-', '13', '-', '14', '-', '15', '-', '17', '-', '18', '-', '19', '-', '20', '-', '20', '-', '22', '-', '22', '-', '22', '-', '23', '-', '23', '-', '24', '-', '24', '-', '25', '-', '25', '-', '25', '-', '25', '-', '26', '-', '26', '-', '28', '-', '29', '-', '29', '-', '32', '-', '36', '-', '37', '-', '43', '-', '43', '-', '47', '-', '47', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '50', '-', '66', '-', '50', '-', '50', '-', '64', '-', '64', '-', '66', '-', '66', '-', '64', '-', '64', '-', '64', '-', '64', '-', '66', '-', '66', '-', '64', '-', '66', '-', '66', '-', '66', '-', '66', '-', '66', '-', '66', '-', '64', '-', '66', '-', '66', '-', '64', '-', '66', '-', '66', '-', '64', '-', '64', '-', '64', '-', '69', '-', '64', '-', '66', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-', '69', '-']\n",
            "Batch    201 in the current validation dataloader | Bleu score  34.65\n",
            "Randomly sampled reference InChI is ['C', '24', 'H', '15', 'N', '3', 'S', '/c', '1', '-', '2', '-', '6', '-', '16', '(', '7', '-', '3', '-', '1', ')', '20', '-', '14', '-', '26', '-', '22', '(', '15', '-', '25', '-', '20', ')', '27', '-', '21', '-', '9', '-', '5', '-', '4', '-', '8', '-', '18', '(', '21', ')', '19', '-', '11', '-', '10', '-', '17', '-', '12', '-', '13', '-', '28', '-', '24', '(', '17', ')', '23', '(', '19', ')', '27', '/h', '1', '-', '15', 'H']\n",
            "the corresponding InChI predicted is ['C', '24', 'H', '21', 'N', '3', 'O', '/c', '1', '-', '16', '-', '8', '-', '5', '-', '6', '-', '10', '-', '20', '(', '17', ')', '26', '-', '23', '(', '28', ')', '24', '(', '16', '-', '28', ')', '26', '-', '15', '-', '15', '-', '27', '(', '16', '-', '16', '-', '28', ')', '23', '(', '28', ')', '17', '-', '8', '-', '3', '-', '2', '-', '4', '-', '9', '-', '19', '(', '18', ')', '12', '-', '9', '-', '14', '-', '24', '/h', '2', '-', '13', ',', '16', 'H', ',', '14', '-', '15', 'H', '2', ',', '1', 'H', '3']\n",
            "Batch    301 in the current validation dataloader | Bleu score  31.48\n",
            "Randomly sampled reference InChI is ['C', '12', 'H', '21', 'N', '3', 'O', '2', '/c', '1', '-', '4', '-', '6', '-', '15', '(', '9', '(', '2', ')', '3', ')', '8', '-', '10', '-', '5', '-', '7', '-', '17', '-', '11', '(', '10', ')', '12', '(', '16', ')', '14', '-', '13', '/h', '5', ',', '7', ',', '9', 'H', ',', '4', ',', '6', ',', '8', ',', '13', 'H', '2', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '14', ',', '16', ')']\n",
            "the corresponding InChI predicted is ['C', '12', 'H', '19', 'N', '3', 'O', '/c', '1', '-', '4', '-', '5', '-', '6', '-', '8', '(', '2', ')', '12', '(', '16', ')', '11', '(', '15', ')', '9', '-', '7', '-', '10', '(', '3', ')', '12', '(', '8', ')', '15', '-', '10', '/h', '7', '-', '8', ',', '11', 'H', ',', '4', '-', '6', 'H', '2', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '13', ',', '15', ')']\n",
            "Batch    401 in the current validation dataloader | Bleu score  32.88\n",
            "Randomly sampled reference InChI is ['C', '11', 'H', '15', 'N', 'O', '2', 'S', '/c', '1', '-', '2', '-', '15', '(', '13', ',', '14', ')', '8', '-', '11', '-', '10', '-', '6', '-', '4', '-', '3', '-', '5', '-', '9', '(', '10', ')', '7', '-', '12', '-', '11', '/h', '3', '-', '6', ',', '11', '-', '12', 'H', ',', '2', ',', '7', '-', '8', 'H', '2', ',', '1', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '11', 'H', '15', 'N', 'O', '2', '/c', '1', '-', '10', '(', '13', ')', '10', '(', '13', ')', '8', '-', '12', '-', '6', '-', '4', '-', '2', '-', '3', '-', '5', '-', '8', '-', '7', '-', '10', '(', '9', ')', '12', '/h', '2', '-', '5', ',', '7', ',', '10', 'H', ',', '6', ',', '8', 'H', '2', ',', '1', 'H', '3']\n",
            "Validation Loss (tensor(0.9590, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1237f80dc0>)\n",
            "Now Training Epoch 1 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   1.02 | Tokens / Sec:  8471.3 | Learning Rate: 1.2e-03\n",
            "Batch    101 in the current epoch | Loss / Token:   1.01 | Tokens / Sec:  7994.5 | Learning Rate: 1.1e-03\n",
            "Batch    201 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  8004.7 | Learning Rate: 1.1e-03\n",
            "Batch    301 in the current epoch | Loss / Token:   1.06 | Tokens / Sec:  8015.7 | Learning Rate: 1.1e-03\n",
            "Batch    401 in the current epoch | Loss / Token:   1.09 | Tokens / Sec:  7995.1 | Learning Rate: 1.1e-03\n",
            "Batch    501 in the current epoch | Loss / Token:   1.02 | Tokens / Sec:  8009.8 | Learning Rate: 1.1e-03\n",
            "Batch    601 in the current epoch | Loss / Token:   1.03 | Tokens / Sec:  8016.0 | Learning Rate: 1.1e-03\n",
            "Batch    701 in the current epoch | Loss / Token:   1.03 | Tokens / Sec:  8035.6 | Learning Rate: 1.1e-03\n",
            "Batch    801 in the current epoch | Loss / Token:   1.04 | Tokens / Sec:  7971.3 | Learning Rate: 1.1e-03\n",
            "Batch    901 in the current epoch | Loss / Token:   1.04 | Tokens / Sec:  7974.0 | Learning Rate: 1.1e-03\n",
            "Batch   1001 in the current epoch | Loss / Token:   1.11 | Tokens / Sec:  8020.9 | Learning Rate: 1.1e-03\n",
            "Batch   1101 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  7961.1 | Learning Rate: 1.1e-03\n",
            "Batch   1201 in the current epoch | Loss / Token:   1.05 | Tokens / Sec:  7990.9 | Learning Rate: 1.1e-03\n",
            "Batch   1301 in the current epoch | Loss / Token:   1.01 | Tokens / Sec:  7968.0 | Learning Rate: 1.0e-03\n",
            "Batch   1401 in the current epoch | Loss / Token:   1.00 | Tokens / Sec:  8045.2 | Learning Rate: 1.0e-03\n",
            "Batch   1501 in the current epoch | Loss / Token:   1.03 | Tokens / Sec:  8011.9 | Learning Rate: 1.0e-03\n",
            "Batch   1601 in the current epoch | Loss / Token:   1.07 | Tokens / Sec:  8000.1 | Learning Rate: 1.0e-03\n",
            "Batch   1701 in the current epoch | Loss / Token:   1.00 | Tokens / Sec:  8004.2 | Learning Rate: 1.0e-03\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.96 | Tokens / Sec:  8009.3 | Learning Rate: 1.0e-03\n",
            "Batch   1901 in the current epoch | Loss / Token:   1.05 | Tokens / Sec:  8024.4 | Learning Rate: 1.0e-03\n",
            "Batch   2001 in the current epoch | Loss / Token:   1.08 | Tokens / Sec:  7965.4 | Learning Rate: 1.0e-03\n",
            "Batch   2101 in the current epoch | Loss / Token:   1.00 | Tokens / Sec:  7953.1 | Learning Rate: 9.9e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   1.00 | Tokens / Sec:  8022.0 | Learning Rate: 9.9e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   1.01 | Tokens / Sec:  7977.6 | Learning Rate: 9.8e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  8008.7 | Learning Rate: 9.7e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   1.05 | Tokens / Sec:  7985.4 | Learning Rate: 9.7e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   1.02 | Tokens / Sec:  7953.1 | Learning Rate: 9.6e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  7989.7 | Learning Rate: 9.6e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   1.02 | Tokens / Sec:  8012.5 | Learning Rate: 9.5e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   1.04 | Tokens / Sec:  7996.1 | Learning Rate: 9.5e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   1.03 | Tokens / Sec:  8013.7 | Learning Rate: 9.4e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   1.04 | Tokens / Sec:  8040.6 | Learning Rate: 9.4e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   1.02 | Tokens / Sec:  8044.9 | Learning Rate: 9.3e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.97 | Tokens / Sec:  8010.7 | Learning Rate: 9.2e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.93 | Tokens / Sec:  8018.2 | Learning Rate: 9.2e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  8003.0 | Learning Rate: 9.1e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   1.01 | Tokens / Sec:  8011.4 | Learning Rate: 9.1e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  7980.4 | Learning Rate: 9.1e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  8012.0 | Learning Rate: 9.0e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  8014.2 | Learning Rate: 9.0e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  7989.6 | Learning Rate: 8.9e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  8000.0 | Learning Rate: 8.9e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.96 | Tokens / Sec:  7984.4 | Learning Rate: 8.8e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.94 | Tokens / Sec:  7983.0 | Learning Rate: 8.8e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.97 | Tokens / Sec:  8027.7 | Learning Rate: 8.7e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.99 | Tokens / Sec:  7988.9 | Learning Rate: 8.7e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  8005.8 | Learning Rate: 8.7e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  7960.5 | Learning Rate: 8.6e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.94 | Tokens / Sec:  7974.7 | Learning Rate: 8.6e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  7990.8 | Learning Rate: 8.5e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   1.00 | Tokens / Sec:  8065.7 | Learning Rate: 8.5e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.93 | Tokens / Sec:  7973.1 | Learning Rate: 8.5e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  8043.8 | Learning Rate: 8.4e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.96 | Tokens / Sec:  8007.0 | Learning Rate: 8.4e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  7986.5 | Learning Rate: 8.3e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.93 | Tokens / Sec:  7968.0 | Learning Rate: 8.3e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  8003.1 | Learning Rate: 8.3e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   1.00 | Tokens / Sec:  7992.6 | Learning Rate: 8.2e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  7982.3 | Learning Rate: 8.2e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 96% | 55% |\n",
            "Now Validating Epoch 1 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  38.03\n",
            "Randomly sampled reference InChI is ['C', '23', 'H', '26', 'N', '2', 'O', '5', 'S', '/c', '1', '-', '5', '-', '18', '-', '12', '-', '19', '(', '23', '(', '27', ')', '28', '-', '6', '-', '2', ')', '22', '(', '31', '-', '18', ')', '24', '-', '21', '(', '26', ')', '11', '-', '16', '-', '7', '-', '9', '-', '17', '(', '10', '-', '8', '-', '16', ')', '29', '-', '13', '-', '20', '-', '14', '(', '3', ')', '25', '-', '30', '-', '15', '(', '20', ')', '4', '/h', '7', '-', '10', ',', '12', 'H', ',', '5', '-', '6', ',', '11', ',', '13', 'H', '2', ',', '1', '-', '4', 'H', '3', ',', '(', 'H', ',', '24', ',', '26', ')']\n",
            "the corresponding InChI predicted is ['C', '24', 'H', '28', 'N', 'O', '6', '/c', '1', '-', '6', '-', '30', '-', '20', '-', '11', '-', '8', '-', '16', '(', '8', '-', '11', '-', '18', ')', '23', '(', '28', ')', '24', '(', '28', ')', '26', '-', '19', '-', '12', '-', '16', '(', '2', ')', '21', '(', '27', ')', '13', '-', '17', '(', '24', ')', '13', '-', '17', '-', '7', '-', '5', '-', '8', '-', '18', '(', '16', ')', '28', '-', '3', '/h', '7', '-', '9', ',', '11', '-', '12', ',', '15', ',', '19', 'H', ',', '6', ',', '10', ',', '13', 'H', '2', ',', '1', '-', '5', 'H', '3', ',', '(', 'H', ',', '24', ',', '27', ')', '/t', '17', '-', ',', '19', '+', '/m', '0', '/s', '1']\n",
            "Batch    101 in the current validation dataloader | Bleu score  39.60\n",
            "Randomly sampled reference InChI is ['C', '12', 'H', '12', 'Br', 'Cl', 'N', '2', 'O', '/c', '1', '-', '7', '(', '2', ')', '5', '-', '9', '-', '10', '(', '13', ')', '11', '(', '14', ')', '16', '-', '12', '(', '15', '-', '9', ')', '8', '-', '3', '-', '4', '-', '17', '-', '6', '-', '8', '/h', '3', '-', '4', ',', '6', '-', '7', 'H', ',', '5', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '12', 'H', '12', 'Cl', 'N', '2', 'O', '/c', '1', '-', '7', '-', '3', '-', '4', '-', '9', '(', '14', '-', '7', ')', '11', '(', '15', ')', '14', '-', '10', '-', '5', '-', '9', '(', '14', '-', '2', ')', '6', '-', '16', '-', '11', '(', '9', ')', '15', '/h', '3', '-', '4', ',', '6', ',', '8', 'H', ',', '5', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "Batch    201 in the current validation dataloader | Bleu score  38.23\n",
            "Randomly sampled reference InChI is ['C', '11', 'H', '12', 'F', 'N', 'O', '5', 'S', '/c', '12', '-', '9', '-', '2', '-', '1', '-', '7', '(', '5', '-', '10', '(', '9', ')', '19', '(', '13', ',', '15', ')', '16', ')', '11', '(', '14', ')', '18', '-', '8', '-', '3', '-', '4', '-', '17', '-', '6', '-', '8', '/h', '1', '-', '2', ',', '5', ',', '8', 'H', ',', '3', '-', '4', ',', '6', 'H', '2', ',', '(', 'H', '2', ',', '13', ',', '15', ',', '16', ')']\n",
            "the corresponding InChI predicted is ['C', '11', 'H', '10', 'Cl', 'N', 'O', '5', 'S', '/c', '12', '-', '9', '-', '3', '-', '1', '-', '2', '-', '9', '(', '9', ')', '15', '-', '10', '(', '15', ')', '6', '-', '15', '-', '5', '-', '4', '-', '8', '(', '12', ')', '7', '-', '15', '-', '11', '(', '14', ')', '15', '/h', '1', '-', '3', ',', '6', 'H', ',', '4', '-', '5', 'H', '2', ',', '(', 'H', ',', '13', ',', '15', ')']\n",
            "Batch    301 in the current validation dataloader | Bleu score  37.97\n",
            "Randomly sampled reference InChI is ['C', '25', 'H', '26', 'F', '3', 'N', '5', 'O', '4', '/c', '26', '-', '25', '(', '27', ',', '28', ')', '22', '-', '7', '-', '3', '-', '6', '-', '18', '(', '29', '-', '22', ')', '24', '(', '35', ')', '30', '-', '20', '-', '12', '-', '16', '-', '14', '-', '33', '(', '15', '-', '23', '(', '34', ')', '32', '-', '8', '-', '10', '-', '36', '-', '11', '-', '9', '-', '32', ')', '31', '-', '19', '(', '16', ')', '13', '-', '21', '(', '20', ')', '37', '-', '17', '-', '4', '-', '1', '-', '2', '-', '5', '-', '17', '/h', '3', ',', '6', '-', '7', ',', '12', '-', '14', ',', '17', 'H', ',', '1', '-', '2', ',', '4', '-', '5', ',', '8', '-', '11', ',', '15', 'H', '2', ',', '(', 'H', ',', '30', ',', '35', ')']\n",
            "the corresponding InChI predicted is ['C', '27', 'H', '29', 'F', '3', 'N', '5', 'O', '2', '/c', '1', '-', '17', '-', '6', '-', '7', '-', '19', '(', '8', '-', '7', '-', '17', ')', '32', '-', '26', '(', '34', ')', '31', '-', '23', '(', '18', '-', '9', '-', '10', '-', '21', '(', '32', '-', '2', ')', '11', '-', '10', '-', '19', ')', '32', '-', '24', '(', '34', ')', '14', '-', '19', '-', '12', '-', '19', '(', '13', '-', '19', ')', '32', '-', '23', '(', '32', '-', '30', ')', '16', '-', '4', '-', '3', '-', '5', '-', '18', '/h', '3', '-', '9', ',', '12', '-', '14', ',', '17', ',', '20', 'H', ',', '10', '-', '11', ',', '14', '-', '15', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '31', ',', '33', ')']\n",
            "Batch    401 in the current validation dataloader | Bleu score  38.53\n",
            "Randomly sampled reference InChI is ['C', '20', 'H', '20', 'Cl', 'N', '3', 'O', '4', 'S', '3', '/c', '1', '-', '3', '-', '27', '-', '19', '(', '26', ')', '16', '-', '11', '-', '5', '-', '4', '-', '10', '(', '2', ')', '8', '-', '13', '(', '11', ')', '31', '-', '18', '(', '16', ')', '22', '-', '15', '(', '25', ')', '9', '-', '29', '-', '20', '-', '24', '-', '23', '-', '17', '(', '28', '-', '20', ')', '12', '-', '6', '-', '7', '-', '14', '(', '21', ')', '30', '-', '12', '/h', '6', '-', '7', ',', '10', 'H', ',', '3', '-', '5', ',', '8', '-', '9', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '22', ',', '25', ')']\n",
            "the corresponding InChI predicted is ['C', '21', 'H', '23', 'Cl', 'N', '3', 'O', '4', 'S', '/c', '1', '-', '3', '-', '28', '-', '18', '(', '27', ')', '14', '-', '8', '-', '12', '(', '2', ')', '19', '(', '27', ')', '24', '-', '19', '(', '27', ')', '24', '-', '17', '(', '12', '-', '6', '-', '4', '-', '5', '-', '12', '(', '10', '-', '12', ')', '24', '-', '17', ')', '26', '-', '17', '(', '26', ')', '11', '-', '7', '-', '9', '-', '27', '-', '11', '/h', '6', '-', '7', ',', '9', '-', '10', ',', '12', ',', '15', 'H', ',', '3', '-', '5', ',', '8', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '24', ',', '26', ')', '/t', '13', '-', ',', '15', '+', '/m', '0', '/s', '1']\n",
            "Validation Loss (tensor(0.8297, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1217b9ffd0>)\n",
            "Now Training Epoch 2 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  8558.1 | Learning Rate: 8.2e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.94 | Tokens / Sec:  8028.1 | Learning Rate: 8.1e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.97 | Tokens / Sec:  8050.4 | Learning Rate: 8.1e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.94 | Tokens / Sec:  7991.3 | Learning Rate: 8.1e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.97 | Tokens / Sec:  7992.3 | Learning Rate: 8.0e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  8021.2 | Learning Rate: 8.0e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.96 | Tokens / Sec:  7993.6 | Learning Rate: 8.0e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  8032.5 | Learning Rate: 7.9e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.97 | Tokens / Sec:  7991.4 | Learning Rate: 7.9e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7996.2 | Learning Rate: 7.9e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   1.03 | Tokens / Sec:  8027.4 | Learning Rate: 7.9e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  7940.2 | Learning Rate: 7.8e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7985.9 | Learning Rate: 7.8e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7982.9 | Learning Rate: 7.8e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.93 | Tokens / Sec:  8026.3 | Learning Rate: 7.7e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.98 | Tokens / Sec:  8009.4 | Learning Rate: 7.7e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.93 | Tokens / Sec:  8008.9 | Learning Rate: 7.7e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7992.8 | Learning Rate: 7.6e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  8034.7 | Learning Rate: 7.6e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   1.00 | Tokens / Sec:  8017.6 | Learning Rate: 7.6e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.94 | Tokens / Sec:  7970.4 | Learning Rate: 7.6e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  7970.9 | Learning Rate: 7.5e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  7993.3 | Learning Rate: 7.5e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  7966.8 | Learning Rate: 7.5e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  8019.3 | Learning Rate: 7.5e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  8001.6 | Learning Rate: 7.4e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   1.01 | Tokens / Sec:  7969.2 | Learning Rate: 7.4e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  7990.0 | Learning Rate: 7.4e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.97 | Tokens / Sec:  8046.6 | Learning Rate: 7.3e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7974.0 | Learning Rate: 7.3e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  8007.0 | Learning Rate: 7.3e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  8064.4 | Learning Rate: 7.3e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.93 | Tokens / Sec:  8051.9 | Learning Rate: 7.2e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  8011.5 | Learning Rate: 7.2e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  8030.6 | Learning Rate: 7.2e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8000.5 | Learning Rate: 7.2e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8027.5 | Learning Rate: 7.2e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7989.6 | Learning Rate: 7.1e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8009.9 | Learning Rate: 7.1e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7994.9 | Learning Rate: 7.1e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8030.1 | Learning Rate: 7.1e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  7980.5 | Learning Rate: 7.0e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  7966.7 | Learning Rate: 7.0e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  8004.7 | Learning Rate: 7.0e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  8023.8 | Learning Rate: 7.0e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7952.9 | Learning Rate: 7.0e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  8018.4 | Learning Rate: 6.9e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.95 | Tokens / Sec:  7980.0 | Learning Rate: 6.9e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7982.2 | Learning Rate: 6.9e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7969.1 | Learning Rate: 6.9e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  8017.1 | Learning Rate: 6.8e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  7960.8 | Learning Rate: 6.8e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8050.9 | Learning Rate: 6.8e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  7985.3 | Learning Rate: 6.8e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  7962.8 | Learning Rate: 6.8e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  7953.0 | Learning Rate: 6.7e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8009.6 | Learning Rate: 6.7e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   0.93 | Tokens / Sec:  8026.9 | Learning Rate: 6.7e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7969.4 | Learning Rate: 6.7e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 96% | 55% |\n",
            "Now Validating Epoch 2 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  40.73\n",
            "Randomly sampled reference InChI is ['C', '18', 'H', '21', 'N', 'O', '2', '/c', '1', '-', '13', '-', '7', '-', '5', '-', '6', '-', '8', '-', '16', '(', '13', ')', '19', '(', '18', '(', '2', ',', '3', ')', '4', ')', '15', '-', '11', '-', '9', '-', '14', '(', '10', '-', '12', '-', '15', ')', '17', '(', '20', ')', '21', '/h', '5', '-', '12', 'H', ',', '1', '-', '4', 'H', '3', ',', '(', 'H', ',', '20', ',', '21', ')']\n",
            "the corresponding InChI predicted is ['C', '18', 'H', '21', 'N', 'O', '2', '/c', '1', '-', '13', '(', '2', ')', '19', '(', '3', ')', '16', '(', '20', ')', '14', '-', '9', '-', '7', '-', '12', '-', '16', '(', '20', ')', '15', '-', '10', '-', '6', '-', '5', '-', '8', '-', '14', '(', '15', ')', '12', '-', '17', '(', '19', ')', '21', '/h', '4', '-', '12', 'H', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '20', ',', '22', ')']\n",
            "Batch    101 in the current validation dataloader | Bleu score  39.61\n",
            "Randomly sampled reference InChI is ['C', '20', 'H', '20', 'F', 'N', 'O', '5', '/c', '1', '-', '25', '-', '14', '-', '9', '-', '7', '-', '13', '(', '8', '-', '10', '-', '14', ')', '19', '(', '23', ')', '22', '-', '12', '-', '15', '(', '11', '-', '17', '(', '22', ')', '20', '(', '24', ')', '26', '-', '2', ')', '27', '-', '18', '-', '6', '-', '4', '-', '3', '-', '5', '-', '16', '(', '18', ')', '21', '/h', '3', '-', '10', ',', '15', ',', '17', 'H', ',', '11', '-', '12', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '20', 'H', '19', 'F', 'N', 'O', '5', '/c', '1', '-', '24', '-', '18', '(', '24', ')', '15', '-', '8', '-', '6', '-', '13', '(', '7', '-', '9', '-', '15', ')', '19', '(', '24', ')', '22', '-', '11', '-', '16', '(', '23', ')', '20', '(', '25', ')', '14', '-', '5', '-', '3', '-', '4', '-', '13', '(', '11', '-', '14', ')', '19', '(', '23', ')', '25', '-', '2', '/h', '3', '-', '10', ',', '13', 'H', ',', '11', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "Batch    201 in the current validation dataloader | Bleu score  42.21\n",
            "Randomly sampled reference InChI is ['C', '10', 'H', '15', 'N', '5', '/c', '1', '-', '8', '(', '2', ')', '15', '-', '10', '(', '11', '-', '7', '-', '13', '-', '15', ')', '6', '-', '14', '-', '9', '(', '3', ')', '4', '-', '5', '-', '12', '-', '14', '/h', '4', '-', '5', ',', '7', '-', '8', 'H', ',', '6', 'H', '2', ',', '1', '-', '3', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '10', 'H', '15', 'N', '5', '/c', '1', '-', '8', '(', '2', ')', '14', '-', '6', '-', '10', '-', '11', '-', '7', '-', '14', '(', '3', ')', '9', '-', '4', '-', '5', '-', '14', '(', '10', ')', '13', '-', '9', '/h', '4', '-', '5', ',', '7', 'H', ',', '6', 'H', '2', ',', '1', '-', '3', 'H', '3']\n",
            "Batch    301 in the current validation dataloader | Bleu score  42.20\n",
            "Randomly sampled reference InChI is ['C', '13', 'H', '18', 'O', '3', '/c', '1', '-', '8', '-', '6', '-', '10', '(', '12', '(', '15', ')', '9', '(', '8', ')', '2', ')', '7', '-', '11', '(', '14', ')', '16', '-', '13', '(', '3', ',', '4', ')', '5', '/h', '7', 'H', ',', '6', 'H', '2', ',', '1', '-', '5', 'H', '3', '/b', '10', '-', '7', '-']\n",
            "the corresponding InChI predicted is ['C', '13', 'H', '18', 'O', '3', '/c', '1', '-', '8', '-', '6', '-', '9', '(', '2', ')', '11', '(', '15', ')', '11', '(', '8', ')', '12', '(', '15', ')', '14', '-', '7', '-', '11', '(', '3', ',', '4', ')', '5', '/h', '6', '-', '7', 'H', ',', '1', '-', '5', 'H', '3']\n",
            "Batch    401 in the current validation dataloader | Bleu score  42.65\n",
            "Randomly sampled reference InChI is ['C', '11', 'H', '19', 'N', '3', '/c', '1', '-', '3', '-', '11', '-', '13', '-', '8', '(', '2', ')', '10', '-', '5', '-', '4', '-', '9', '(', '6', '-', '12', ')', '7', '-', '14', '(', '10', ')', '11', '/h', '9', 'H', ',', '3', '-', '7', ',', '12', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '11', 'H', '19', 'N', '3', '/c', '1', '-', '3', '-', '10', '-', '7', '-', '13', '-', '11', '(', '14', '-', '9', ')', '9', '-', '4', '-', '5', '-', '12', '-', '6', '-', '9', '(', '9', ')', '2', '/h', '8', '-', '10', ',', '12', 'H', ',', '3', '-', '7', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "Validation Loss (tensor(0.7623, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1237f80880>)\n",
            "Now Training Epoch 3 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  8557.9 | Learning Rate: 6.7e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8013.4 | Learning Rate: 6.7e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  8005.4 | Learning Rate: 6.6e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7984.5 | Learning Rate: 6.6e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  7984.1 | Learning Rate: 6.6e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  8022.8 | Learning Rate: 6.6e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  8009.4 | Learning Rate: 6.6e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8011.3 | Learning Rate: 6.6e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7970.1 | Learning Rate: 6.5e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7940.0 | Learning Rate: 6.5e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8020.4 | Learning Rate: 6.5e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  7960.1 | Learning Rate: 6.5e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  7986.8 | Learning Rate: 6.5e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7989.4 | Learning Rate: 6.4e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  8017.6 | Learning Rate: 6.4e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7959.3 | Learning Rate: 6.4e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7992.5 | Learning Rate: 6.4e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7999.6 | Learning Rate: 6.4e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  8017.4 | Learning Rate: 6.4e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8009.6 | Learning Rate: 6.3e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  7960.1 | Learning Rate: 6.3e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7952.7 | Learning Rate: 6.3e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8007.8 | Learning Rate: 6.3e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  7953.2 | Learning Rate: 6.3e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  8000.9 | Learning Rate: 6.3e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  7997.7 | Learning Rate: 6.2e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  7940.5 | Learning Rate: 6.2e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7978.4 | Learning Rate: 6.2e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7993.6 | Learning Rate: 6.2e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7965.0 | Learning Rate: 6.2e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7972.1 | Learning Rate: 6.2e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8033.5 | Learning Rate: 6.2e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  8022.0 | Learning Rate: 6.1e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8000.1 | Learning Rate: 6.1e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  8000.6 | Learning Rate: 6.1e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8015.0 | Learning Rate: 6.1e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7989.9 | Learning Rate: 6.1e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  7966.3 | Learning Rate: 6.1e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  7993.8 | Learning Rate: 6.1e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7986.0 | Learning Rate: 6.0e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7991.8 | Learning Rate: 6.0e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7953.9 | Learning Rate: 6.0e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  7978.8 | Learning Rate: 6.0e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7993.4 | Learning Rate: 6.0e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  8013.5 | Learning Rate: 6.0e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7950.1 | Learning Rate: 6.0e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  8005.4 | Learning Rate: 5.9e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7979.7 | Learning Rate: 5.9e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  7982.2 | Learning Rate: 5.9e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7953.1 | Learning Rate: 5.9e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  8017.0 | Learning Rate: 5.9e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7972.2 | Learning Rate: 5.9e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  8044.2 | Learning Rate: 5.9e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  7992.5 | Learning Rate: 5.9e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7958.6 | Learning Rate: 5.8e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7948.8 | Learning Rate: 5.8e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7990.3 | Learning Rate: 5.8e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7985.0 | Learning Rate: 5.8e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.91 | Tokens / Sec:  7964.0 | Learning Rate: 5.8e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 95% | 55% |\n",
            "Now Validating Epoch 3 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  42.72\n",
            "Randomly sampled reference InChI is ['C', '13', 'H', '16', 'N', '4', 'O', '2', 'S', '/c', '1', '-', '3', '-', '10', '-', '6', '-', '4', '-', '5', '-', '7', '-', '11', '(', '10', ')', '14', '-', '12', '-', '8', '-', '9', '-', '13', '(', '16', '-', '15', '-', '12', ')', '17', '-', '20', '(', '2', ',', '18', ')', '19', '/h', '4', '-', '9', 'H', ',', '3', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '14', ',', '15', ')', '(', 'H', ',', '16', ',', '17', ')']\n",
            "the corresponding InChI predicted is ['C', '13', 'H', '15', 'N', '4', 'O', '2', 'S', '/c', '1', '-', '3', '-', '11', '-', '6', '-', '4', '-', '5', '-', '7', '-', '11', '(', '11', ')', '15', '-', '13', '(', '18', ')', '16', '-', '11', '-', '8', '-', '9', '-', '18', '(', '2', ')', '14', '-', '11', '/h', '4', '-', '9', 'H', ',', '3', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '14', ',', '17', ')', '(', 'H', ',', '15', ',', '17', ')']\n",
            "Batch    101 in the current validation dataloader | Bleu score  42.98\n",
            "Randomly sampled reference InChI is ['C', '31', 'H', '20', 'N', '6', '/c', '1', '-', '2', '-', '11', '-', '25', '-', '23', '(', '10', '-', '1', ')', '24', '-', '19', '-', '21', '(', '14', '-', '15', '-', '26', '(', '24', ')', '34', '-', '25', ')', '20', '-', '8', '-', '7', '-', '9', '-', '22', '(', '18', '-', '20', ')', '29', '-', '35', '-', '30', '(', '27', '-', '12', '-', '3', '-', '5', '-', '16', '-', '32', '-', '27', ')', '37', '-', '31', '(', '36', '-', '29', ')', '28', '-', '13', '-', '4', '-', '6', '-', '17', '-', '33', '-', '28', '/h', '1', '-', '19', ',', '34', 'H']\n",
            "the corresponding InChI predicted is ['C', '31', 'H', '20', 'N', '6', '/c', '1', '-', '2', '-', '11', '-', '31', '-', '31', '(', '35', '-', '23', '-', '11', '-', '4', '-', '3', '-', '5', '-', '12', '-', '24', ')', '31', '-', '31', '-', '31', '(', '35', '-', '30', ')', '19', '-', '20', '-', '7', '-', '8', '-', '20', '(', '9', '-', '8', '-', '19', ')', '28', '-', '19', '-', '33', '-', '26', '-', '14', '-', '6', '-', '12', '-', '24', '(', '18', '-', '23', ')', '31', '-', '19', '-', '19', '-', '33', '-', '19', '-', '18', '-', '35', '-', '27', '/h', '2', '-', '17', 'H', ',', '1', 'H', '3', ',', '(', 'H', ',', '33', ',', '35', ')']\n",
            "Batch    201 in the current validation dataloader | Bleu score  42.42\n",
            "Randomly sampled reference InChI is ['C', '22', 'H', '24', 'Cl', 'N', 'O', '7', 'S', '/c', '1', '-', '22', '(', '2', ',', '3', ')', '24', '-', '32', '(', '27', ',', '28', ')', '19', '-', '13', '-', '15', '(', '8', '-', '9', '-', '16', '(', '19', ')', '23', ')', '21', '(', '26', ')', '31', '-', '17', '-', '10', '-', '6', '-', '14', '(', '12', '-', '18', '(', '17', ')', '29', '-', '4', ')', '7', '-', '11', '-', '20', '(', '25', ')', '30', '-', '5', '/h', '6', '-', '13', ',', '24', 'H', ',', '1', '-', '5', 'H', '3', '/b', '11', '-', '7', '+']\n",
            "the corresponding InChI predicted is ['C', '22', 'H', '23', 'Cl', 'N', 'O', '7', 'S', '/c', '1', '-', '21', '(', '2', ',', '3', ')', '30', '-', '19', '-', '10', '-', '8', '-', '15', '(', '9', '-', '10', '-', '18', '(', '19', ')', '28', '-', '4', ')', '12', '-', '18', '(', '24', ')', '24', '-', '20', '(', '26', ')', '13', '-', '29', '-', '19', '-', '7', '-', '5', '-', '14', '(', '24', ')', '6', '-', '12', '-', '18', '(', '18', ')', '28', '-', '21', '/h', '5', '-', '12', ',', '23', 'H', ',', '1', '-', '4', 'H', '3']\n",
            "Batch    301 in the current validation dataloader | Bleu score  43.76\n",
            "Randomly sampled reference InChI is ['C', '22', 'H', '21', 'Br', 'N', '4', 'O', '2', 'S', '/c', '1', '-', '3', '-', '29', '-', '20', '(', '28', ')', '18', '-', '14', '(', '2', ')', '24', '-', '21', '-', '25', '-', '22', '(', '30', '-', '13', '-', '15', '-', '7', '-', '5', '-', '4', '-', '6', '-', '8', '-', '15', ')', '26', '-', '27', '(', '21', ')', '19', '(', '18', ')', '16', '-', '9', '-', '11', '-', '17', '(', '23', ')', '12', '-', '10', '-', '16', '/h', '4', '-', '12', ',', '19', 'H', ',', '3', ',', '13', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '24', ',', '25', ',', '26', ')']\n",
            "the corresponding InChI predicted is ['C', '21', 'H', '20', 'Cl', 'N', '3', 'O', '2', 'S', '/c', '1', '-', '3', '-', '28', '-', '19', '(', '26', ')', '17', '-', '13', '(', '2', ')', '24', '-', '20', '(', '27', ')', '17', '(', '14', '-', '8', '-', '5', '-', '4', '-', '6', '-', '9', '-', '14', ')', '24', '-', '20', '(', '26', ')', '17', '-', '11', '-', '7', '-', '10', '-', '16', '(', '22', ')', '19', '(', '16', ')', '28', '-', '12', '-', '11', '-', '27', '/h', '4', '-', '11', ',', '14', 'H', ',', '3', ',', '12', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '24', ',', '26', ')']\n",
            "Batch    401 in the current validation dataloader | Bleu score  40.36\n",
            "Randomly sampled reference InChI is ['C', '10', 'H', '16', 'N', '2', 'O', '2', '/c', '1', '-', '5', '-', '14', '-', '10', '(', '13', ')', '9', '-', '6', '-', '11', '-', '12', '(', '7', '(', '2', ')', '3', ')', '8', '(', '9', ')', '4', '/h', '6', '-', '7', 'H', ',', '5', 'H', '2', ',', '1', '-', '4', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '10', 'H', '15', 'N', '2', 'O', '2', '/c', '1', '-', '5', '-', '14', '-', '9', '(', '12', ')', '8', '(', '6', '-', '11', '-', '4', ')', '7', '(', '2', ')', '11', '-', '10', '(', '13', ')', '14', '-', '3', '/h', '7', 'H', ',', '5', '-', '6', 'H', '2', ',', '1', '-', '4', 'H', '3']\n",
            "Validation Loss (tensor(0.7165, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1237f809d0>)\n",
            "Now Training Epoch 4 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  8563.4 | Learning Rate: 5.8e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7994.0 | Learning Rate: 5.8e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7988.6 | Learning Rate: 5.8e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7975.1 | Learning Rate: 5.7e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  7952.8 | Learning Rate: 5.7e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7989.5 | Learning Rate: 5.7e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7991.5 | Learning Rate: 5.7e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  8016.8 | Learning Rate: 5.7e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.94 | Tokens / Sec:  7969.7 | Learning Rate: 5.7e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7927.8 | Learning Rate: 5.7e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7989.3 | Learning Rate: 5.7e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7962.2 | Learning Rate: 5.7e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7976.5 | Learning Rate: 5.6e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7967.1 | Learning Rate: 5.6e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8005.4 | Learning Rate: 5.6e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  7973.1 | Learning Rate: 5.6e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7982.9 | Learning Rate: 5.6e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  7975.7 | Learning Rate: 5.6e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.89 | Tokens / Sec:  8004.9 | Learning Rate: 5.6e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  8003.0 | Learning Rate: 5.6e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  7965.8 | Learning Rate: 5.6e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7958.3 | Learning Rate: 5.5e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  8005.1 | Learning Rate: 5.5e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.88 | Tokens / Sec:  7972.1 | Learning Rate: 5.5e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7988.8 | Learning Rate: 5.5e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7977.9 | Learning Rate: 5.5e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  7955.7 | Learning Rate: 5.5e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7977.1 | Learning Rate: 5.5e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.92 | Tokens / Sec:  7993.3 | Learning Rate: 5.5e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7958.6 | Learning Rate: 5.5e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7973.6 | Learning Rate: 5.4e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  8038.8 | Learning Rate: 5.4e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  8039.2 | Learning Rate: 5.4e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  8003.6 | Learning Rate: 5.4e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7998.1 | Learning Rate: 5.4e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7992.0 | Learning Rate: 5.4e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7993.6 | Learning Rate: 5.4e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7960.8 | Learning Rate: 5.4e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  8006.2 | Learning Rate: 5.4e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7987.5 | Learning Rate: 5.4e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7996.2 | Learning Rate: 5.3e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7957.6 | Learning Rate: 5.3e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7952.8 | Learning Rate: 5.3e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7952.2 | Learning Rate: 5.3e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  7976.3 | Learning Rate: 5.3e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.86 | Tokens / Sec:  7949.9 | Learning Rate: 5.3e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7987.1 | Learning Rate: 5.3e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7960.8 | Learning Rate: 5.3e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7955.1 | Learning Rate: 5.3e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7955.7 | Learning Rate: 5.3e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8030.0 | Learning Rate: 5.3e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7960.4 | Learning Rate: 5.2e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8052.7 | Learning Rate: 5.2e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7990.0 | Learning Rate: 5.2e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7964.8 | Learning Rate: 5.2e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7964.8 | Learning Rate: 5.2e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7987.1 | Learning Rate: 5.2e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  8002.9 | Learning Rate: 5.2e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7948.6 | Learning Rate: 5.2e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 96% | 55% |\n",
            "Now Validating Epoch 4 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  44.75\n",
            "Randomly sampled reference InChI is ['C', '21', 'H', '29', 'N', '3', 'O', '3', '/c', '1', '-', '15', '-', '14', '-', '18', '(', '21', '(', '26', ')', '24', '(', '15', ')', '17', '-', '6', '-', '3', '-', '2', '-', '4', '-', '7', '-', '17', ')', '22', '-', '20', '(', '25', ')', '19', '-', '8', '-', '5', '-', '11', '-', '23', '(', '19', ')', '16', '-', '9', '-', '12', '-', '27', '-', '13', '-', '10', '-', '16', '/h', '2', '-', '4', ',', '6', '-', '7', ',', '15', '-', '16', ',', '18', '-', '19', 'H', ',', '5', ',', '8', '-', '14', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', ',', '22', ',', '25', ')', '/t', '15', '-', ',', '18', '-', ',', '19', '-', '/m', '1', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '20', 'H', '26', 'N', '3', 'O', '3', '/c', '25', '-', '18', '(', '23', ')', '14', '-', '5', '-', '2', '-', '1', '-', '3', '-', '6', '-', '15', '(', '14', ')', '20', '(', '25', ')', '22', '-', '11', '-', '10', '-', '19', '(', '25', ')', '23', '-', '17', '-', '8', '-', '4', '-', '7', '-', '16', '(', '17', ')', '22', '-', '12', '-', '9', '-', '21', '/h', '1', '-', '3', ',', '5', '-', '6', ',', '14', '-', '15', ',', '17', ',', '21', 'H', ',', '4', ',', '7', '-', '13', 'H', '2', ',', '(', 'H', ',', '21', ',', '24', ')', '/t', '15', '-', ',', '17', '+', '/m', '1', '/s', '1']\n",
            "Batch    101 in the current validation dataloader | Bleu score  45.15\n",
            "Randomly sampled reference InChI is ['C', '23', 'H', '28', 'N', '2', 'O', '3', 'S', '/c', '1', '-', '17', '-', '13', '-', '15', '-', '25', '(', '16', '-', '14', '-', '17', ')', '29', '(', '27', ',', '28', ')', '20', '-', '11', '-', '9', '-', '19', '(', '10', '-', '12', '-', '20', ')', '23', '(', '26', ')', '24', '-', '22', '-', '8', '-', '4', '-', '6', '-', '18', '-', '5', '-', '2', '-', '3', '-', '7', '-', '21', '(', '18', ')', '22', '/h', '2', '-', '3', ',', '5', ',', '7', ',', '9', '-', '12', ',', '17', ',', '22', 'H', ',', '4', ',', '6', ',', '8', ',', '13', '-', '16', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', ',', '24', ',', '26', ')', '/t', '22', '-', '/m', '1', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '23', 'H', '27', 'N', '2', 'O', '3', 'S', '/c', '1', '-', '16', '-', '7', '-', '11', '-', '20', '(', '12', '-', '8', '-', '17', ')', '26', '-', '23', '(', '27', ')', '23', '(', '27', ')', '24', '-', '15', '-', '13', '-', '20', '(', '14', '-', '15', '-', '24', ')', '26', '-', '20', '-', '9', '-', '5', '-', '6', '-', '18', '-', '4', '-', '2', '-', '3', '-', '11', '-', '21', '(', '18', ')', '20', '/h', '4', '-', '5', ',', '8', '-', '10', ',', '14', ',', '17', ',', '21', ',', '24', 'H', ',', '2', '-', '3', ',', '6', '-', '7', ',', '11', '-', '13', ',', '15', 'H', '2', ',', '1', 'H', '3']\n",
            "Batch    201 in the current validation dataloader | Bleu score  43.28\n",
            "Randomly sampled reference InChI is ['C', '22', 'H', '18', 'F', '2', 'O', '6', '/c', '23', '-', '13', '-', '3', '-', '11', '(', '4', '-', '14', '(', '24', ')', '6', '-', '13', ')', '10', '-', '29', '-', '21', '-', '9', '-', '16', '-', '18', '(', '27', ')', '7', '-', '15', '(', '25', ')', '8', '-', '20', '(', '16', ')', '30', '-', '22', '(', '21', ')', '12', '-', '1', '-', '2', '-', '17', '(', '26', ')', '19', '(', '28', ')', '5', '-', '12', '/h', '1', '-', '8', ',', '21', '-', '22', ',', '25', '-', '28', 'H', ',', '9', '-', '10', 'H', '2', '/t', '21', '-', ',', '22', '+', '/m', '0', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '23', 'H', '21', 'F', 'O', '6', '/c', '1', '-', '12', '-', '2', '-', '4', '-', '16', '(', '5', '-', '3', '-', '13', ')', '22', '(', '28', ')', '10', '-', '26', '-', '21', '(', '27', ')', '8', '-', '15', '-', '6', '-', '15', '(', '10', '-', '19', '(', '16', ')', '28', ')', '7', '-', '15', '(', '10', '-', '24', ')', '21', '(', '27', ')', '19', '(', '24', ')', '11', '-', '28', '-', '13', '/h', '2', '-', '7', ',', '9', '-', '11', ',', '19', ',', '27', 'H', ',', '8', ',', '12', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', ',', '27', ',', '28', ')', '/t', '19', '-', '/m', '0', '/s', '1']\n",
            "Batch    301 in the current validation dataloader | Bleu score  45.33\n",
            "Randomly sampled reference InChI is ['C', '17', 'H', '17', 'N', '3', 'O', '/c', '1', '-', '12', '(', '16', '-', '5', '-', '3', '-', '4', '-', '9', '-', '18', '-', '16', ')', '20', '-', '17', '-', '15', '-', '7', '-', '6', '-', '14', '(', '21', '-', '2', ')', '11', '-', '13', '(', '15', ')', '8', '-', '10', '-', '19', '-', '17', '/h', '3', '-', '12', 'H', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '19', ',', '20', ')']\n",
            "the corresponding InChI predicted is ['C', '17', 'H', '17', 'N', '3', 'O', '/c', '1', '-', '12', '(', '19', '-', '2', ')', '16', '-', '9', '-', '7', '-', '13', '(', '9', '-', '10', '-', '17', ')', '19', '-', '16', '(', '20', '-', '16', ')', '13', '-', '5', '-', '3', '-', '4', '-', '6', '-', '15', '(', '13', ')', '20', '-', '3', '/h', '4', '-', '11', 'H', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '19', ',', '20', ')']\n",
            "Batch    401 in the current validation dataloader | Bleu score  46.06\n",
            "Randomly sampled reference InChI is ['C', '21', 'H', '25', 'N', '3', 'O', '3', '/c', '1', '-', '15', '-', '11', '-', '20', '(', '25', ')', '22', '-', '18', '-', '9', '-', '4', '-', '5', '-', '10', '-', '19', '(', '18', ')', '24', '(', '15', ')', '21', '(', '26', ')', '14', '-', '23', '(', '2', ')', '13', '-', '16', '-', '7', '-', '6', '-', '8', '-', '17', '(', '12', '-', '16', ')', '27', '-', '3', '/h', '4', '-', '10', ',', '12', ',', '15', 'H', ',', '11', ',', '13', '-', '14', 'H', '2', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '22', ',', '25', ')']\n",
            "the corresponding InChI predicted is ['C', '20', 'H', '24', 'N', '3', 'O', '3', '/c', '1', '-', '15', '-', '8', '-', '4', '-', '5', '-', '9', '-', '17', '(', '15', ')', '23', '(', '13', '-', '15', ')', '12', '-', '19', '(', '24', ')', '22', '(', '2', ')', '12', '-', '17', '-', '10', '-', '6', '-', '7', '-', '16', '(', '19', '(', '17', ')', '24', ')', '21', '-', '12', '-', '24', '-', '3', '/h', '4', '-', '10', ',', '13', 'H', ',', '11', '-', '12', ',', '14', 'H', '2', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '21', ',', '25', ')']\n",
            "Validation Loss (tensor(0.6750, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1237f80730>)\n",
            "Now Training Epoch 5 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  8572.7 | Learning Rate: 5.2e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7991.9 | Learning Rate: 5.2e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7999.8 | Learning Rate: 5.2e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7979.8 | Learning Rate: 5.1e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7959.4 | Learning Rate: 5.1e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7994.3 | Learning Rate: 5.1e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7976.5 | Learning Rate: 5.1e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  8025.0 | Learning Rate: 5.1e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.90 | Tokens / Sec:  7983.7 | Learning Rate: 5.1e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7938.3 | Learning Rate: 5.1e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   0.87 | Tokens / Sec:  8010.7 | Learning Rate: 5.1e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7948.3 | Learning Rate: 5.1e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7980.7 | Learning Rate: 5.1e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7934.5 | Learning Rate: 5.1e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7983.6 | Learning Rate: 5.1e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7994.4 | Learning Rate: 5.0e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  8019.7 | Learning Rate: 5.0e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  8000.6 | Learning Rate: 5.0e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  8009.9 | Learning Rate: 5.0e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  8013.8 | Learning Rate: 5.0e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7960.6 | Learning Rate: 5.0e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7939.9 | Learning Rate: 5.0e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7991.7 | Learning Rate: 5.0e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7981.7 | Learning Rate: 5.0e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  8003.9 | Learning Rate: 5.0e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7985.3 | Learning Rate: 5.0e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7947.9 | Learning Rate: 5.0e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7975.8 | Learning Rate: 5.0e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  8017.5 | Learning Rate: 4.9e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7961.5 | Learning Rate: 4.9e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7989.2 | Learning Rate: 4.9e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  8046.3 | Learning Rate: 4.9e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8066.2 | Learning Rate: 4.9e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8004.3 | Learning Rate: 4.9e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8002.8 | Learning Rate: 4.9e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7981.3 | Learning Rate: 4.9e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7989.9 | Learning Rate: 4.9e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7976.7 | Learning Rate: 4.9e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  8020.3 | Learning Rate: 4.9e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7983.9 | Learning Rate: 4.9e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7990.2 | Learning Rate: 4.9e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7983.0 | Learning Rate: 4.8e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7984.5 | Learning Rate: 4.8e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7961.2 | Learning Rate: 4.8e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7983.6 | Learning Rate: 4.8e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7960.3 | Learning Rate: 4.8e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7987.5 | Learning Rate: 4.8e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7964.6 | Learning Rate: 4.8e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7968.9 | Learning Rate: 4.8e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7975.2 | Learning Rate: 4.8e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  8058.6 | Learning Rate: 4.8e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7971.5 | Learning Rate: 4.8e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  8026.4 | Learning Rate: 4.8e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  8003.6 | Learning Rate: 4.8e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7989.4 | Learning Rate: 4.8e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7984.3 | Learning Rate: 4.7e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8000.4 | Learning Rate: 4.7e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7992.2 | Learning Rate: 4.7e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7967.4 | Learning Rate: 4.7e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 95% | 55% |\n",
            "Now Validating Epoch 5 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  47.60\n",
            "Randomly sampled reference InChI is ['C', '18', 'H', '20', 'N', '2', 'O', '4', 'S', '/c', '1', '-', '11', '-', '3', '-', '8', '-', '16', '(', '17', '(', '21', ')', '9', '-', '11', ')', '19', '-', '18', '(', '22', ')', '15', '-', '10', '-', '14', '(', '7', '-', '4', '-', '12', '(', '15', ')', '2', ')', '25', '(', '23', ',', '24', ')', '20', '-', '13', '-', '5', '-', '6', '-', '13', '/h', '3', '-', '4', ',', '7', '-', '10', ',', '13', ',', '20', '-', '21', 'H', ',', '5', '-', '6', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '19', ',', '22', ')']\n",
            "the corresponding InChI predicted is ['C', '18', 'H', '20', 'N', '2', 'O', '4', 'S', '/c', '1', '-', '11', '-', '4', '-', '5', '-', '14', '(', '6', '-', '2', ')', '16', '(', '10', '-', '11', ')', '19', '-', '18', '(', '22', ')', '17', '(', '20', '-', '17', '(', '20', ')', '21', ')', '13', '-', '7', '-', '8', '-', '15', '(', '9', '-', '13', ')', '25', '(', '3', ',', '23', ')', '24', '/h', '4', '-', '9', ',', '11', ',', '19', 'H', ',', '10', 'H', '2', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '20', ',', '22', ')']\n",
            "Batch    101 in the current validation dataloader | Bleu score  47.55\n",
            "Randomly sampled reference InChI is ['C', '12', 'H', '11', 'N', '3', 'S', '/c', '13', '-', '5', '-', '3', '-', '9', '-', '7', '-', '10', '(', '1', '-', '2', '-', '12', '(', '9', ')', '14', ')', '15', '-', '11', '-', '4', '-', '6', '-', '16', '-', '8', '-', '11', '/h', '1', '-', '2', ',', '4', ',', '6', '-', '8', ',', '15', 'H', ',', '3', ',', '14', 'H', '2']\n",
            "the corresponding InChI predicted is ['C', '12', 'H', '10', 'N', '3', 'S', '/c', '13', '-', '11', '-', '3', '-', '1', '-', '9', '(', '2', '-', '4', '-', '14', ')', '12', '(', '10', ')', '15', '-', '11', '-', '5', '-', '7', '-', '16', '-', '8', '-', '10', '/h', '1', '-', '4', ',', '6', '-', '7', ',', '9', 'H', ',', '5', 'H', '2', ',', '(', 'H', '2', ',', '14', ',', '15', ')']\n",
            "Batch    201 in the current validation dataloader | Bleu score  49.17\n",
            "Randomly sampled reference InChI is ['C', '15', 'H', '15', 'Cl', '2', 'N', 'O', '/c', '1', '-', '9', '-', '5', '-', '12', '(', '6', '-', '10', '(', '2', ')', '15', '(', '9', ')', '17', ')', '19', '-', '8', '-', '11', '-', '3', '-', '4', '-', '13', '(', '16', ')', '14', '(', '18', ')', '7', '-', '11', '/h', '3', '-', '7', 'H', ',', '8', ',', '18', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '15', 'H', '15', 'Cl', '2', 'N', 'O', '/c', '1', '-', '9', '-', '3', '-', '4', '-', '12', '(', '7', '-', '17', ')', '8', '-', '19', '-', '15', '-', '6', '-', '11', '(', '9', ')', '5', '-', '12', '(', '15', ')', '14', '(', '16', ')', '10', '(', '2', ')', '17', '/h', '3', '-', '7', 'H', ',', '8', ',', '18', 'H', '2', ',', '1', '-', '2', 'H', '3']\n",
            "Batch    301 in the current validation dataloader | Bleu score  48.48\n",
            "Randomly sampled reference InChI is ['C', '13', 'H', '18', 'Cl', '2', 'N', '2', '/c', '1', '-', '17', '(', '13', '-', '6', '-', '3', '-', '2', '-', '4', '-', '7', '-', '13', ')', '9', '-', '5', '-', '8', '-', '16', '-', '11', '-', '12', '(', '15', ')', '10', '-', '14', '/h', '2', '-', '4', ',', '6', '-', '7', ',', '10', ',', '16', 'H', ',', '5', ',', '8', '-', '9', ',', '11', 'H', '2', ',', '1', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '13', 'H', '18', 'Cl', '2', 'N', '2', '/c', '1', '-', '16', '-', '10', '-', '8', '-', '16', '-', '13', '(', '16', '-', '16', '-', '11', '-', '16', ')', '9', '-', '7', '-', '12', '-', '5', '-', '3', '-', '2', '-', '4', '-', '6', '-', '12', '/h', '2', '-', '6', ',', '9', ',', '11', ',', '16', 'H', ',', '7', '-', '8', ',', '10', 'H', '2', ',', '1', 'H', '3']\n",
            "Batch    401 in the current validation dataloader | Bleu score  45.66\n",
            "Randomly sampled reference InChI is ['C', '27', 'H', '38', 'N', '4', 'O', '5', '/c', '1', '-', '19', '-', '18', '-', '23', '-', '22', '(', '7', '-', '17', '-', '35', '-', '23', ')', '24', '(', '29', '-', '19', ')', '31', '-', '13', '-', '11', '-', '30', '(', '12', '-', '14', '-', '31', ')', '10', '-', '4', '-', '20', '-', '2', '-', '8', '-', '27', '(', '9', '-', '3', '-', '20', ',', '26', '(', '28', ')', '33', ')', '36', '-', '25', '(', '32', ')', '21', '-', '5', '-', '15', '-', '34', '-', '16', '-', '6', '-', '21', '/h', '7', ',', '17', '-', '18', ',', '20', '-', '21', 'H', ',', '2', '-', '6', ',', '8', '-', '16', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', '2', ',', '28', ',', '33', ')']\n",
            "the corresponding InChI predicted is ['C', '26', 'H', '36', 'N', '4', 'O', '5', '/c', '1', '-', '18', '-', '25', '(', '31', ')', '31', '(', '2', ')', '26', '(', '33', ')', '31', '-', '10', '-', '7', '-', '24', '(', '8', '-', '11', '-', '30', ')', '31', '-', '13', '-', '5', '-', '20', '(', '6', '-', '14', '-', '31', ')', '3', '-', '4', '-', '23', '(', '31', ')', '28', '-', '13', '-', '8', '-', '28', '-', '15', '-', '19', '-', '2', '-', '17', '-', '33', '-', '15', '-', '14', '-', '33', '-', '17', '-', '17', '-', '33', '/h', '6', '-', '7', ',', '18', '-', '19', ',', '22', 'H', ',', '3', '-', '5', ',', '8', '-', '17', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '27', ',', '31', ')']\n",
            "Validation Loss (tensor(0.6425, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1237f80520>)\n",
            "Now Training Epoch 6 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  8611.2 | Learning Rate: 4.7e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8005.4 | Learning Rate: 4.7e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8005.2 | Learning Rate: 4.7e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7988.6 | Learning Rate: 4.7e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7993.5 | Learning Rate: 4.7e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8010.1 | Learning Rate: 4.7e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7988.4 | Learning Rate: 4.7e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  8018.2 | Learning Rate: 4.7e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7964.0 | Learning Rate: 4.7e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7954.0 | Learning Rate: 4.7e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  8011.7 | Learning Rate: 4.7e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7946.4 | Learning Rate: 4.7e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7978.6 | Learning Rate: 4.6e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7964.2 | Learning Rate: 4.6e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7999.3 | Learning Rate: 4.6e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7977.7 | Learning Rate: 4.6e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7991.1 | Learning Rate: 4.6e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7976.2 | Learning Rate: 4.6e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  8032.7 | Learning Rate: 4.6e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  8030.3 | Learning Rate: 4.6e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7961.6 | Learning Rate: 4.6e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7936.9 | Learning Rate: 4.6e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  8007.2 | Learning Rate: 4.6e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7962.7 | Learning Rate: 4.6e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7996.3 | Learning Rate: 4.6e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7987.9 | Learning Rate: 4.6e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7944.8 | Learning Rate: 4.6e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7979.9 | Learning Rate: 4.6e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.85 | Tokens / Sec:  7994.8 | Learning Rate: 4.5e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7944.6 | Learning Rate: 4.5e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.81 | Tokens / Sec:  7977.8 | Learning Rate: 4.5e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8039.6 | Learning Rate: 4.5e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  8035.8 | Learning Rate: 4.5e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  8007.2 | Learning Rate: 4.5e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7993.0 | Learning Rate: 4.5e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7979.8 | Learning Rate: 4.5e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7994.9 | Learning Rate: 4.5e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7969.3 | Learning Rate: 4.5e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7986.6 | Learning Rate: 4.5e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7980.3 | Learning Rate: 4.5e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7983.8 | Learning Rate: 4.5e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.84 | Tokens / Sec:  7980.4 | Learning Rate: 4.5e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7976.6 | Learning Rate: 4.5e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7985.1 | Learning Rate: 4.5e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7978.3 | Learning Rate: 4.5e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7944.2 | Learning Rate: 4.4e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7980.9 | Learning Rate: 4.4e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7934.9 | Learning Rate: 4.4e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7958.9 | Learning Rate: 4.4e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7953.0 | Learning Rate: 4.4e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  8011.9 | Learning Rate: 4.4e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  7963.8 | Learning Rate: 4.4e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  8028.5 | Learning Rate: 4.4e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7978.5 | Learning Rate: 4.4e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7940.2 | Learning Rate: 4.4e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7959.6 | Learning Rate: 4.4e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  8022.9 | Learning Rate: 4.4e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8000.2 | Learning Rate: 4.4e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7970.7 | Learning Rate: 4.4e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 96% | 55% |\n",
            "Now Validating Epoch 6 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  48.07\n",
            "Randomly sampled reference InChI is ['C', '23', 'H', '18', 'N', '2', 'O', '2', 'S', '/c', '26', '-', '22', '(', '25', '-', '14', '-', '13', '-', '16', '-', '7', '-', '2', '-', '5', '-', '11', '-', '20', '(', '16', ')', '25', ')', '18', '-', '9', '-', '3', '-', '1', '-', '8', '-', '17', '(', '18', ')', '15', '-', '28', '-', '23', '-', '24', '-', '19', '-', '10', '-', '4', '-', '6', '-', '12', '-', '21', '(', '19', ')', '27', '-', '23', '/h', '1', '-', '12', 'H', ',', '13', '-', '15', 'H', '2']\n",
            "the corresponding InChI predicted is ['C', '23', 'H', '17', 'N', '2', 'O', '3', 'S', '/c', '26', '-', '22', '(', '25', '-', '14', '-', '16', '-', '7', '-', '2', '-', '1', '-', '3', '-', '8', '-', '16', ')', '22', '(', '26', ')', '26', '-', '14', '-', '10', '-', '16', '-', '10', '-', '4', '-', '5', '-', '11', '-', '19', '(', '16', ')', '27', '-', '22', '-', '17', '-', '8', '-', '6', '-', '12', '-', '19', '(', '18', ')', '27', '-', '22', '/h', '1', '-', '13', 'H', ',', '14', '-', '15', 'H', '2']\n",
            "Batch    101 in the current validation dataloader | Bleu score  48.16\n",
            "Randomly sampled reference InChI is ['C', '12', 'H', '17', 'N', 'O', '3', '/c', '1', '-', '2', '-', '3', '-', '4', '-', '7', '-', '13', '-', '11', '(', '14', ')', '9', '-', '5', '-', '6', '-', '10', '(', '8', '-', '9', ')', '12', '(', '15', ')', '16', '/h', '9', '-', '10', 'H', ',', '4', '-', '8', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', ',', '13', ',', '14', ')', '(', 'H', ',', '15', ',', '16', ')']\n",
            "the corresponding InChI predicted is ['C', '12', 'H', '17', 'N', 'O', '3', '/c', '1', '-', '2', '-', '3', '-', '4', '-', '5', '-', '6', '-', '10', '(', '14', ')', '13', '-', '11', '-', '7', '-', '8', '-', '11', '(', '15', ')', '15', '-', '10', '/h', '9', '-', '10', ',', '12', 'H', ',', '2', '-', '8', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', ',', '14', ',', '15', ')', '(', 'H', ',', '15', ',', '16', ')']\n",
            "Batch    201 in the current validation dataloader | Bleu score  47.74\n",
            "Randomly sampled reference InChI is ['C', '27', 'H', '35', 'N', '3', 'O', '5', 'S', '/c', '1', '-', '19', '(', '2', ')', '9', '-', '10', '-', '22', '-', '11', '-', '12', '-', '26', '-', '24', '(', '14', '-', '22', ')', '35', '-', '25', '(', '20', '(', '3', ')', '16', '-', '30', '(', '21', '(', '4', ')', '18', '-', '31', ')', '36', '(', '26', ',', '33', ')', '34', ')', '17', '-', '29', '(', '5', ')', '27', '(', '32', ')', '15', '-', '23', '-', '8', '-', '6', '-', '7', '-', '13', '-', '28', '-', '23', '/h', '6', '-', '8', ',', '11', '-', '14', ',', '19', '-', '21', ',', '25', ',', '31', 'H', ',', '15', '-', '18', 'H', '2', ',', '1', '-', '5', 'H', '3', '/t', '20', '-', ',', '21', '+', ',', '25', '-', '/m', '0', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '28', 'H', '37', 'N', '4', 'O', '4', '/c', '1', '-', '19', '(', '2', ')', '15', '-', '28', '(', '33', ')', '31', '(', '17', '-', '20', '(', '3', ')', '4', ')', '26', '(', '33', ')', '28', '(', '33', ')', '31', '(', '16', '-', '21', '-', '10', '-', '7', '-', '8', '-', '10', '-', '22', ')', '25', '(', '16', '-', '22', ')', '29', '-', '24', '-', '12', '-', '11', '-', '22', '(', '13', '-', '13', '-', '23', ')', '32', '(', '5', ')', '6', '/h', '7', '-', '14', ',', '17', ',', '19', ',', '22', ',', '25', ',', '30', 'H', ',', '15', ',', '17', '-', '18', 'H', '2', ',', '1', '-', '6', 'H', '3', '/t', '21', '-', ',', '24', '+', '/m', '1', '/s', '1']\n",
            "Batch    301 in the current validation dataloader | Bleu score  48.75\n",
            "Randomly sampled reference InChI is ['C', '20', 'H', '19', 'F', '3', 'N', '2', 'O', '2', 'S', '/c', '1', '-', '3', '-', '13', '-', '25', '-', '18', '(', '20', '(', '21', ',', '22', ')', '23', ')', '19', '(', '14', '-', '24', '-', '25', ',', '15', '-', '9', '-', '5', '-', '4', '-', '6', '-', '10', '-', '15', ')', '16', '-', '11', '-', '7', '-', '8', '-', '12', '-', '17', '(', '16', ')', '28', '(', '2', ',', '26', ')', '27', '/h', '3', '-', '12', ',', '14', ',', '18', 'H', ',', '1', ',', '13', 'H', '2', ',', '2', 'H', '3']\n",
            "the corresponding InChI predicted is ['C', '20', 'H', '19', 'F', '3', 'N', '2', 'O', '2', 'S', '/c', '1', '-', '3', '-', '13', '-', '25', '-', '20', '(', '26', ')', '18', '(', '15', '-', '10', '-', '5', '-', '2', '-', '6', '-', '11', '-', '15', ')', '19', '(', '25', ')', '24', '-', '14', '-', '16', '(', '22', ')', '17', '-', '12', '-', '8', '-', '7', '-', '9', '-', '13', '-', '18', '(', '17', ')', '22', '/h', '2', '-', '12', ',', '16', 'H', ',', '13', 'H', '2', ',', '1', 'H', '3', '/t', '17', '-', '/m', '1', '/s', '1']\n",
            "Batch    401 in the current validation dataloader | Bleu score  47.21\n",
            "Randomly sampled reference InChI is ['C', '23', 'H', '22', 'N', '4', 'O', '3', 'S', '/c', '1', '-', '16', '-', '8', '-', '9', '-', '18', '(', '12', '-', '21', '(', '16', ')', '30', '-', '11', '-', '10', '-', '29', '-', '19', '-', '6', '-', '4', '-', '3', '-', '5', '-', '7', '-', '19', ')', '26', '-', '23', '-', '27', '-', '20', '(', '15', '-', '31', '-', '23', ')', '17', '-', '13', '-', '24', '-', '22', '(', '28', '-', '2', ')', '25', '-', '14', '-', '17', '/h', '3', '-', '9', ',', '12', '-', '15', 'H', ',', '10', '-', '11', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '26', ',', '27', ')']\n",
            "the corresponding InChI predicted is ['C', '23', 'H', '22', 'N', '4', 'O', '3', 'S', '/c', '1', '-', '16', '-', '12', '-', '19', '(', '28', '-', '2', ')', '20', '(', '14', '-', '28', '-', '14', '-', '16', ')', '26', '-', '22', '(', '27', ')', '14', '-', '29', '-', '21', '-', '10', '-', '8', '-', '18', '(', '9', '-', '11', '-', '29', '-', '21', ')', '24', '-', '22', '(', '27', ')', '16', '-', '6', '-', '4', '-', '3', '-', '5', '-', '7', '-', '16', '/h', '3', '-', '11', ',', '13', '-', '14', 'H', ',', '12', ',', '15', 'H', '2', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '24', ',', '26', ')']\n",
            "Validation Loss (tensor(0.6213, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1237f815a0>)\n",
            "Now Training Epoch 7 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  8610.0 | Learning Rate: 4.4e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8001.9 | Learning Rate: 4.4e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7992.8 | Learning Rate: 4.4e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7962.6 | Learning Rate: 4.4e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7981.5 | Learning Rate: 4.4e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  8001.1 | Learning Rate: 4.3e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7988.8 | Learning Rate: 4.3e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  8017.9 | Learning Rate: 4.3e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7988.2 | Learning Rate: 4.3e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.66 | Tokens / Sec:  7945.3 | Learning Rate: 4.3e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7973.0 | Learning Rate: 4.3e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7918.8 | Learning Rate: 4.3e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7982.9 | Learning Rate: 4.3e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7955.4 | Learning Rate: 4.3e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  8010.7 | Learning Rate: 4.3e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7971.0 | Learning Rate: 4.3e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7989.4 | Learning Rate: 4.3e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8001.6 | Learning Rate: 4.3e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  8029.4 | Learning Rate: 4.3e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  8032.7 | Learning Rate: 4.3e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7978.5 | Learning Rate: 4.3e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7941.8 | Learning Rate: 4.3e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8016.2 | Learning Rate: 4.3e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7956.5 | Learning Rate: 4.3e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7984.5 | Learning Rate: 4.3e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7973.4 | Learning Rate: 4.2e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  7950.7 | Learning Rate: 4.2e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7985.7 | Learning Rate: 4.2e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.65 | Tokens / Sec:  7983.9 | Learning Rate: 4.2e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7966.0 | Learning Rate: 4.2e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7990.7 | Learning Rate: 4.2e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.82 | Tokens / Sec:  8061.0 | Learning Rate: 4.2e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  8031.1 | Learning Rate: 4.2e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7987.3 | Learning Rate: 4.2e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7987.9 | Learning Rate: 4.2e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7988.0 | Learning Rate: 4.2e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  8005.7 | Learning Rate: 4.2e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7969.5 | Learning Rate: 4.2e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7983.8 | Learning Rate: 4.2e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7978.1 | Learning Rate: 4.2e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  8015.3 | Learning Rate: 4.2e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7980.6 | Learning Rate: 4.2e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7980.2 | Learning Rate: 4.2e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7973.0 | Learning Rate: 4.2e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7986.5 | Learning Rate: 4.2e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7960.8 | Learning Rate: 4.2e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7994.7 | Learning Rate: 4.1e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7989.0 | Learning Rate: 4.1e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7977.3 | Learning Rate: 4.1e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7959.5 | Learning Rate: 4.1e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  8032.3 | Learning Rate: 4.1e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7959.0 | Learning Rate: 4.1e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  8043.1 | Learning Rate: 4.1e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7985.9 | Learning Rate: 4.1e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7968.0 | Learning Rate: 4.1e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7981.7 | Learning Rate: 4.1e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7999.0 | Learning Rate: 4.1e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  8025.9 | Learning Rate: 4.1e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7973.5 | Learning Rate: 4.1e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 96% | 55% |\n",
            "Now Validating Epoch 7 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  47.21\n",
            "Randomly sampled reference InChI is ['C', '32', 'H', '30', 'N', '2', 'O', '5', 'S', '/c', '1', '-', '4', '-', '5', '-', '14', '-', '38', '-', '23', '-', '10', '-', '7', '-', '20', '(', '8', '-', '11', '-', '23', ')', '28', '-', '27', '(', '29', '(', '35', ')', '21', '-', '9', '-', '13', '-', '25', '-', '22', '(', '17', '-', '21', ')', '16', '-', '19', '(', '3', ')', '39', '-', '25', ')', '30', '(', '36', ')', '31', '(', '37', ')', '34', '(', '28', ')', '32', '-', '33', '-', '24', '-', '12', '-', '6', '-', '18', '(', '2', ')', '15', '-', '26', '(', '24', ')', '40', '-', '32', '/h', '6', '-', '13', ',', '15', ',', '17', ',', '19', ',', '28', ',', '35', 'H', ',', '4', '-', '5', ',', '14', ',', '16', 'H', '2', ',', '1', '-', '3', 'H', '3', '/b', '29', '-', '27', '+', '/t', '19', '-', ',', '28', '+', '/m', '1', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '32', 'H', '31', 'N', '3', 'O', '6', 'S', '/c', '1', '-', '4', '-', '5', '-', '6', '-', '7', '-', '21', '-', '14', '-', '23', '(', '15', '-', '21', ')', '31', '(', '38', ')', '35', '-', '28', '(', '19', '-', '8', '-', '10', '-', '21', '(', '10', '-', '9', '-', '19', ')', '38', ')', '28', '(', '36', ')', '31', '(', '38', ')', '33', '-', '28', '(', '36', ')', '32', '(', '39', ')', '35', '(', '18', '-', '20', '-', '12', '-', '13', '-', '23', '(', '38', '-', '2', ')', '15', '-', '21', ')', '17', '-', '24', '(', '3', ')', '38', '/h', '8', '-', '13', ',', '16', '-', '17', ',', '24', ',', '28', ',', '33', 'H', ',', '4', '-', '7', ',', '14', '-', '14', 'H', '2', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '33', ',', '38', ')', '/b', '28', '-', '17', '-', '/t', '26', '-', '/m', '0', '/s', '1']\n",
            "Batch    101 in the current validation dataloader | Bleu score  51.53\n",
            "Randomly sampled reference InChI is ['C', '27', 'H', '35', 'Br', 'N', '2', 'O', '3', '/c', '1', '-', '18', '-', '10', '-', '12', '-', '22', '(', '13', '-', '11', '-', '18', ')', '16', '-', '30', '(', '21', '(', '4', ')', '27', '(', '32', ')', '29', '-', '23', '-', '8', '-', '6', '-', '5', '-', '7', '-', '9', '-', '23', ')', '25', '(', '31', ')', '17', '-', '33', '-', '24', '-', '14', '-', '19', '(', '2', ')', '26', '(', '28', ')', '20', '(', '3', ')', '15', '-', '24', '/h', '10', '-', '15', ',', '21', ',', '23', 'H', ',', '5', '-', '9', ',', '16', '-', '17', 'H', '2', ',', '1', '-', '4', 'H', '3', ',', '(', 'H', ',', '29', ',', '32', ')', '/t', '21', '-', '/m', '0', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '27', 'H', '34', 'Cl', 'N', '2', 'O', '3', '/c', '1', '-', '18', '-', '8', '-', '10', '-', '22', '(', '11', '-', '9', '-', '19', ')', '28', '(', '32', ')', '30', '(', '16', '-', '21', '-', '12', '-', '13', '-', '24', '(', '29', ')', '14', '-', '13', '-', '21', ')', '26', '(', '31', ')', '17', '-', '31', '(', '17', '-', '21', '-', '6', '-', '5', '-', '4', '-', '7', '-', '21', ')', '26', '(', '32', ')', '17', '(', '2', ')', '3', '/h', '8', '-', '11', ',', '14', '-', '16', ',', '19', ',', '23', 'H', ',', '4', '-', '7', ',', '12', '-', '13', ',', '17', 'H', '2', ',', '1', '-', '3', 'H', '3', ',', '(', 'H', ',', '29', ',', '32', ')', '/t', '20', '-', ',', '23', '+', '/m', '1', '/s', '1']\n",
            "Batch    201 in the current validation dataloader | Bleu score  49.80\n",
            "Randomly sampled reference InChI is ['C', '11', 'H', '12', 'O', '3', '/c', '1', '-', '13', '-', '10', '-', '6', '-', '5', '-', '9', '(', '4', '-', '3', '-', '7', '-', '12', ')', '11', '(', '8', '-', '10', ')', '14', '-', '2', '/h', '3', '-', '8', 'H', ',', '1', '-', '2', 'H', '3', '/b', '4', '-', '3', '+']\n",
            "the corresponding InChI predicted is ['C', '11', 'H', '13', 'O', '3', '/c', '1', '-', '14', '-', '11', '-', '6', '-', '4', '-', '9', '(', '7', '-', '11', ')', '5', '-', '3', '-', '9', '(', '2', ')', '8', '-', '12', '/h', '3', '-', '8', 'H', ',', '1', '-', '2', 'H', '3', '/b', '6', '-', '5', '+']\n",
            "Batch    301 in the current validation dataloader | Bleu score  51.19\n",
            "Randomly sampled reference InChI is ['C', '23', 'H', '19', 'Cl', 'N', '4', 'O', 'S', '/c', '1', '-', '14', '-', '3', '-', '9', '-', '19', '-', '20', '(', '11', '-', '12', '-', '25', '-', '23', '(', '19', ')', '26', '-', '14', ')', '28', '-', '21', '-', '13', '-', '16', '(', '24', ')', '4', '-', '10', '-', '22', '(', '21', ')', '30', '-', '18', '-', '7', '-', '5', '-', '17', '(', '6', '-', '8', '-', '18', ')', '27', '-', '15', '(', '2', ')', '29', '/h', '3', '-', '13', 'H', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '27', ',', '29', ')', '(', 'H', ',', '25', ',', '26', ',', '28', ')']\n",
            "the corresponding InChI predicted is ['C', '23', 'H', '19', 'Cl', 'N', '4', 'O', 'S', '/c', '1', '-', '14', '-', '11', '-', '16', '(', '2', ')', '28', '-', '23', '(', '28', ')', '19', '-', '6', '-', '4', '-', '16', '(', '5', '-', '7', '-', '19', ')', '24', '-', '20', '-', '8', '-', '10', '-', '22', '(', '28', '-', '22', ')', '28', '-', '14', '-', '17', '-', '3', '-', '10', '-', '20', '(', '26', '-', '13', '-', '16', ')', '29', '-', '13', '-', '16', '/h', '3', '-', '13', 'H', ',', '1', '-', '2', 'H', '3', ',', '(', 'H', ',', '25', ',', '28', ')', '(', 'H', ',', '26', ',', '27', ')']\n",
            "Batch    401 in the current validation dataloader | Bleu score  51.39\n",
            "Randomly sampled reference InChI is ['C', '26', 'H', '21', 'N', '3', 'O', '2', 'S', '/c', '30', '-', '18', '-', '10', '-', '6', '-', '9', '-', '17', '(', '13', '-', '18', ')', '24', '-', '23', '-', '20', '(', '19', '-', '11', '-', '4', '-', '5', '-', '12', '-', '21', '(', '19', ')', '27', '-', '23', ')', '14', '-', '22', '-', '25', '(', '31', ')', '28', '(', '26', '(', '32', ')', '29', '(', '22', ')', '24', ')', '15', '-', '16', '-', '7', '-', '2', '-', '1', '-', '3', '-', '8', '-', '16', '/h', '1', '-', '13', ',', '24', ',', '27', ',', '30', '-', '31', 'H', ',', '14', '-', '15', 'H', '2']\n",
            "the corresponding InChI predicted is ['C', '26', 'H', '21', 'N', '3', 'O', '2', 'S', '/c', '31', '-', '23', '-', '14', '-', '19', '(', '13', '-', '19', '(', '23', ')', '30', ')', '26', '(', '31', ')', '29', '(', '25', '(', '30', ')', '30', ')', '16', '-', '9', '-', '4', '-', '5', '-', '10', '-', '19', '(', '16', ')', '28', '(', '25', ')', '15', '-', '18', '-', '11', '-', '7', '-', '12', '-', '20', '(', '30', ')', '28', '(', '15', '-', '17', ')', '17', '-', '6', '-', '2', '-', '1', '-', '3', '-', '7', '-', '16', '/h', '1', '-', '12', ',', '22', ',', '29', '-', '30', 'H', ',', '13', '-', '15', 'H', '2']\n",
            "Validation Loss (tensor(0.5982, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d1217b9fc40>)\n",
            "Now Training Epoch 8 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  8526.3 | Learning Rate: 4.1e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  8013.9 | Learning Rate: 4.1e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  8024.8 | Learning Rate: 4.1e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7979.4 | Learning Rate: 4.1e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7970.1 | Learning Rate: 4.1e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  8002.3 | Learning Rate: 4.1e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  8001.8 | Learning Rate: 4.1e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8022.8 | Learning Rate: 4.1e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7956.8 | Learning Rate: 4.1e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7941.7 | Learning Rate: 4.1e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7992.8 | Learning Rate: 4.0e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7920.8 | Learning Rate: 4.0e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7992.8 | Learning Rate: 4.0e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7977.4 | Learning Rate: 4.0e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  8007.8 | Learning Rate: 4.0e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7972.9 | Learning Rate: 4.0e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  8005.5 | Learning Rate: 4.0e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7991.7 | Learning Rate: 4.0e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  8022.3 | Learning Rate: 4.0e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  8009.1 | Learning Rate: 4.0e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7955.3 | Learning Rate: 4.0e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.67 | Tokens / Sec:  7942.8 | Learning Rate: 4.0e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7995.8 | Learning Rate: 4.0e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7974.2 | Learning Rate: 4.0e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  8001.2 | Learning Rate: 4.0e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7979.9 | Learning Rate: 4.0e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7965.9 | Learning Rate: 4.0e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7999.3 | Learning Rate: 4.0e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  8010.0 | Learning Rate: 4.0e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7947.7 | Learning Rate: 4.0e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7989.9 | Learning Rate: 4.0e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.80 | Tokens / Sec:  8043.6 | Learning Rate: 4.0e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  8044.4 | Learning Rate: 4.0e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7998.0 | Learning Rate: 4.0e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  8006.8 | Learning Rate: 3.9e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7979.6 | Learning Rate: 3.9e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.67 | Tokens / Sec:  7990.6 | Learning Rate: 3.9e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7964.9 | Learning Rate: 3.9e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7982.9 | Learning Rate: 3.9e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  8007.3 | Learning Rate: 3.9e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7996.4 | Learning Rate: 3.9e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.67 | Tokens / Sec:  7965.3 | Learning Rate: 3.9e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7990.8 | Learning Rate: 3.9e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7961.5 | Learning Rate: 3.9e-04\n",
            "Batch   4401 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7992.7 | Learning Rate: 3.9e-04\n",
            "Batch   4501 in the current epoch | Loss / Token:   0.79 | Tokens / Sec:  7968.6 | Learning Rate: 3.9e-04\n",
            "Batch   4601 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7992.7 | Learning Rate: 3.9e-04\n",
            "Batch   4701 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7956.7 | Learning Rate: 3.9e-04\n",
            "Batch   4801 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7947.6 | Learning Rate: 3.9e-04\n",
            "Batch   4901 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7944.3 | Learning Rate: 3.9e-04\n",
            "Batch   5001 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  8038.7 | Learning Rate: 3.9e-04\n",
            "Batch   5101 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  7960.0 | Learning Rate: 3.9e-04\n",
            "Batch   5201 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  8032.8 | Learning Rate: 3.9e-04\n",
            "Batch   5301 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7984.2 | Learning Rate: 3.9e-04\n",
            "Batch   5401 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7951.8 | Learning Rate: 3.9e-04\n",
            "Batch   5501 in the current epoch | Loss / Token:   0.67 | Tokens / Sec:  7968.0 | Learning Rate: 3.9e-04\n",
            "Batch   5601 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  8001.1 | Learning Rate: 3.9e-04\n",
            "Batch   5701 in the current epoch | Loss / Token:   0.65 | Tokens / Sec:  7999.8 | Learning Rate: 3.9e-04\n",
            "Batch   5801 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7953.0 | Learning Rate: 3.9e-04\n",
            "| ID | GPU | MEM |\n",
            "------------------\n",
            "|  0 | 95% | 55% |\n",
            "Now Validating Epoch 8 ====\n",
            "Batch      1 in the current validation dataloader | Bleu score  49.36\n",
            "Randomly sampled reference InChI is ['C', '35', 'H', '61', 'N', '5', 'O', '4', '/c', '1', '-', '6', '-', '7', '-', '8', '-', '9', '-', '10', '-', '11', '-', '12', '-', '13', '-', '14', '-', '15', '-', '16', '-', '17', '-', '18', '-', '19', '-', '20', '-', '21', '-', '31', '(', '41', ')', '44', '-', '25', '-', '30', '(', '22', '-', '23', '-', '43', '-', '34', '(', '42', ')', '28', '(', '4', ')', '27', '(', '2', ')', '3', ')', '24', '-', '40', '-', '26', '-', '37', '-', '32', '-', '29', '(', '5', ')', '38', '-', '35', '(', '36', ')', '39', '-', '33', '(', '32', ')', '40', '/h', '26', '-', '28', ',', '30', 'H', ',', '6', '-', '25', 'H', '2', ',', '1', '-', '5', 'H', '3', ',', '(', 'H', '2', ',', '36', ',', '38', ',', '39', ')', '/t', '28', '-', ',', '30', '+', '/m', '0', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '38', 'H', '60', 'N', '2', 'O', '4', '/c', '1', '-', '5', '-', '7', '-', '8', '-', '9', '-', '10', '-', '11', '-', '12', '-', '13', '-', '14', '-', '14', '-', '15', '-', '16', '-', '16', '-', '17', '-', '18', '-', '18', '-', '19', '-', '19', '-', '20', '-', '20', '-', '21', '-', '21', '-', '22', '-', '26', '-', '30', '-', '31', '-', '32', '(', '38', ')', '35', '(', '42', ')', '38', '(', '42', ')', '38', '(', '3', ',', '4', ')', '38', '-', '31', '(', '41', ')', '35', '(', '42', ')', '38', '(', '2', ')', '6', '-', '2', '/h', '22', '-', '23', ',', '25', '-', '26', ',', '29', ',', '34', ',', '38', 'H', ',', '5', '-', '20', ',', '22', '-', '23', ',', '24', '-', '25', 'H', '2', ',', '1', '-', '4', 'H', '3', ',', '(', 'H', ',', '38', ',', '40', ')']\n",
            "Batch    101 in the current validation dataloader | Bleu score  50.07\n",
            "Randomly sampled reference InChI is ['C', '16', 'H', '29', 'N', '3', 'O', '/c', '20', '-', '16', '(', '12', '-', '14', '-', '4', '-', '3', '-', '7', '-', '17', '-', '8', '-', '6', '-', '14', ')', '19', '-', '11', '-', '10', '-', '18', '-', '9', '-', '2', '-', '1', '-', '5', '-', '15', '(', '18', ')', '13', '-', '19', '/h', '14', '-', '15', ',', '17', 'H', ',', '1', '-', '13', 'H', '2']\n",
            "the corresponding InChI predicted is ['C', '16', 'H', '29', 'N', '3', 'O', '/c', '20', '-', '16', '(', '12', '-', '15', '-', '4', '-', '2', '-', '1', '-', '3', '-', '5', '-', '15', ')', '13', '-', '18', '-', '9', '-', '7', '-', '17', '-', '15', '(', '8', '-', '10', '-', '18', ')', '6', '-', '11', '-', '20', '/h', '14', '-', '15', ',', '17', 'H', ',', '1', '-', '13', 'H', '2']\n",
            "Batch    201 in the current validation dataloader | Bleu score  47.72\n",
            "Randomly sampled reference InChI is ['C', '19', 'H', '24', 'N', '2', 'O', '2', '/c', '1', '-', '13', '-', '5', '-', '6', '-', '15', '-', '16', '(', '12', '-', '20', '-', '17', '(', '15', ')', '11', '-', '13', ')', '14', '-', '7', '-', '9', '-', '21', '(', '10', '-', '8', '-', '14', ')', '18', '(', '22', ')', '23', '-', '19', '(', '2', ',', '3', ')', '4', '/h', '5', '-', '7', ',', '11', '-', '12', ',', '16', 'H', ',', '8', '-', '10', 'H', '2', ',', '1', '-', '4', 'H', '3', '/t', '16', '-', '/m', '0', '/s', '1']\n",
            "the corresponding InChI predicted is ['C', '19', 'H', '24', 'N', '2', 'O', '2', 'S', '/c', '1', '-', '13', '-', '4', '-', '6', '-', '15', '(', '7', '-', '5', '-', '13', ')', '19', '(', '3', ',', '4', ')', '19', '(', '2', ',', '5', '-', '9', '-', '20', '-', '18', ')', '17', '(', '22', ')', '14', '-', '10', '-', '11', '-', '22', '-', '12', '-', '15', '/h', '6', '-', '7', ',', '10', ',', '12', '-', '13', 'H', ',', '8', '-', '9', ',', '11', 'H', '2', ',', '1', '-', '5', 'H', '3', '/t', '13', '-', '/m', '1', '/s', '1']\n",
            "Batch    301 in the current validation dataloader | Bleu score  53.27\n",
            "Randomly sampled reference InChI is ['C', '14', 'H', '21', 'N', '3', 'O', '3', '/c', '1', '-', '5', '-', '11', '-', '7', '-', '20', '-', '8', '(', '2', ')', '6', '-', '17', '(', '11', ')', '13', '-', '12', '(', '14', '(', '18', ')', '19', ')', '9', '(', '3', ')', '10', '(', '4', ')', '15', '-', '16', '-', '13', '/h', '8', ',', '11', 'H', ',', '5', '-', '7', 'H', '2', ',', '1', '-', '4', 'H', '3', ',', '(', 'H', ',', '18', ',', '19', ')']\n",
            "the corresponding InChI predicted is ['C', '14', 'H', '21', 'N', '3', 'O', '3', '/c', '1', '-', '5', '-', '11', '-', '12', '(', '16', '-', '18', ')', '7', '-', '18', '(', '6', '-', '11', '(', '11', ')', '3', ')', '14', '(', '19', ')', '15', '-', '8', '(', '2', ')', '10', '(', '4', ')', '14', '(', '18', ')', '19', '/h', '8', '-', '9', 'H', ',', '5', '-', '7', 'H', '2', ',', '1', '-', '4', 'H', '3', ',', '(', 'H', ',', '17', ',', '19', ')']\n",
            "Batch    401 in the current validation dataloader | Bleu score  50.21\n",
            "Randomly sampled reference InChI is ['C', '16', 'H', '18', 'N', '2', 'O', 'S', '/c', '1', '-', '2', '-', '12', '-', '6', '-', '5', '-', '7', '-', '13', '(', '10', '-', '12', ')', '18', '-', '16', '(', '19', ')', '11', '-', '20', '-', '15', '-', '9', '-', '4', '-', '3', '-', '8', '-', '14', '(', '15', ')', '17', '/h', '3', '-', '10', 'H', ',', '2', ',', '11', ',', '17', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', ',', '18', ',', '19', ')']\n",
            "the corresponding InChI predicted is ['C', '16', 'H', '18', 'N', '2', 'O', 'S', '/c', '1', '-', '2', '-', '12', '-', '6', '-', '4', '-', '3', '-', '5', '-', '13', '(', '12', ')', '17', '-', '16', '(', '19', ')', '11', '-', '20', '-', '15', '-', '9', '-', '7', '-', '13', '(', '17', ')', '8', '-', '10', '-', '15', '/h', '3', '-', '10', 'H', ',', '2', ',', '11', ',', '17', 'H', '2', ',', '1', 'H', '3', ',', '(', 'H', ',', '18', ',', '19', ')']\n",
            "Validation Loss (tensor(0.5830, device='cuda:0'), <utils_InChI_translate.Train_Counter object at 0x7d11dd9ba8c0>)\n",
            "Now Training Epoch 9 ====\n",
            "Batch      1 in the current epoch | Loss / Token:   0.66 | Tokens / Sec:  8610.0 | Learning Rate: 3.9e-04\n",
            "Batch    101 in the current epoch | Loss / Token:   0.66 | Tokens / Sec:  7992.5 | Learning Rate: 3.9e-04\n",
            "Batch    201 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  8007.4 | Learning Rate: 3.8e-04\n",
            "Batch    301 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7959.8 | Learning Rate: 3.8e-04\n",
            "Batch    401 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7976.3 | Learning Rate: 3.8e-04\n",
            "Batch    501 in the current epoch | Loss / Token:   0.72 | Tokens / Sec:  7994.4 | Learning Rate: 3.8e-04\n",
            "Batch    601 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7984.4 | Learning Rate: 3.8e-04\n",
            "Batch    701 in the current epoch | Loss / Token:   0.67 | Tokens / Sec:  8009.0 | Learning Rate: 3.8e-04\n",
            "Batch    801 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7960.8 | Learning Rate: 3.8e-04\n",
            "Batch    901 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7949.1 | Learning Rate: 3.8e-04\n",
            "Batch   1001 in the current epoch | Loss / Token:   0.83 | Tokens / Sec:  7990.5 | Learning Rate: 3.8e-04\n",
            "Batch   1101 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7945.7 | Learning Rate: 3.8e-04\n",
            "Batch   1201 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7979.4 | Learning Rate: 3.8e-04\n",
            "Batch   1301 in the current epoch | Loss / Token:   0.64 | Tokens / Sec:  7945.6 | Learning Rate: 3.8e-04\n",
            "Batch   1401 in the current epoch | Loss / Token:   0.66 | Tokens / Sec:  8006.9 | Learning Rate: 3.8e-04\n",
            "Batch   1501 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  8011.5 | Learning Rate: 3.8e-04\n",
            "Batch   1601 in the current epoch | Loss / Token:   0.66 | Tokens / Sec:  7983.6 | Learning Rate: 3.8e-04\n",
            "Batch   1701 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7969.3 | Learning Rate: 3.8e-04\n",
            "Batch   1801 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7997.7 | Learning Rate: 3.8e-04\n",
            "Batch   1901 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  8029.6 | Learning Rate: 3.8e-04\n",
            "Batch   2001 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7965.6 | Learning Rate: 3.8e-04\n",
            "Batch   2101 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7943.0 | Learning Rate: 3.8e-04\n",
            "Batch   2201 in the current epoch | Loss / Token:   0.67 | Tokens / Sec:  7994.5 | Learning Rate: 3.8e-04\n",
            "Batch   2301 in the current epoch | Loss / Token:   0.74 | Tokens / Sec:  7969.4 | Learning Rate: 3.8e-04\n",
            "Batch   2401 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  8000.3 | Learning Rate: 3.8e-04\n",
            "Batch   2501 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  7995.3 | Learning Rate: 3.8e-04\n",
            "Batch   2601 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  7947.0 | Learning Rate: 3.8e-04\n",
            "Batch   2701 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7979.6 | Learning Rate: 3.8e-04\n",
            "Batch   2801 in the current epoch | Loss / Token:   0.71 | Tokens / Sec:  8003.7 | Learning Rate: 3.8e-04\n",
            "Batch   2901 in the current epoch | Loss / Token:   0.70 | Tokens / Sec:  7952.9 | Learning Rate: 3.8e-04\n",
            "Batch   3001 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7944.7 | Learning Rate: 3.8e-04\n",
            "Batch   3101 in the current epoch | Loss / Token:   0.78 | Tokens / Sec:  8053.8 | Learning Rate: 3.7e-04\n",
            "Batch   3201 in the current epoch | Loss / Token:   0.75 | Tokens / Sec:  8056.0 | Learning Rate: 3.7e-04\n",
            "Batch   3301 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7991.7 | Learning Rate: 3.7e-04\n",
            "Batch   3401 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7983.2 | Learning Rate: 3.7e-04\n",
            "Batch   3501 in the current epoch | Loss / Token:   0.64 | Tokens / Sec:  7999.2 | Learning Rate: 3.7e-04\n",
            "Batch   3601 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  8012.0 | Learning Rate: 3.7e-04\n",
            "Batch   3701 in the current epoch | Loss / Token:   0.77 | Tokens / Sec:  7968.5 | Learning Rate: 3.7e-04\n",
            "Batch   3801 in the current epoch | Loss / Token:   0.76 | Tokens / Sec:  7987.4 | Learning Rate: 3.7e-04\n",
            "Batch   3901 in the current epoch | Loss / Token:   0.73 | Tokens / Sec:  7979.7 | Learning Rate: 3.7e-04\n",
            "Batch   4001 in the current epoch | Loss / Token:   0.68 | Tokens / Sec:  7986.9 | Learning Rate: 3.7e-04\n",
            "Batch   4101 in the current epoch | Loss / Token:   0.64 | Tokens / Sec:  7978.2 | Learning Rate: 3.7e-04\n",
            "Batch   4201 in the current epoch | Loss / Token:   0.69 | Tokens / Sec:  7975.0 | Learning Rate: 3.7e-04\n",
            "Batch   4301 in the current epoch | Loss / Token:   0.66 | Tokens / Sec:  7987.3 | Learning Rate: 3.7e-04\n"
          ]
        }
      ],
      "source": [
        "#Train the model, cross validate it, and save it as InChItrainslationfinalmodel.pt.\n",
        "#Note that run_one_epoch in utils_InChI has been modified to show the live translation, Bleu score, and validation loss.\n",
        "gpu = 0\n",
        "print(f\"Train worker process using GPU: {gpu} for training\", flush = True)\n",
        "torch.manual_seed(CFG.seed)\n",
        "torch.cuda.set_device(gpu)\n",
        "loss_func = utils.loss_function(CFG.vocab_length, tokenizer.text2index['<pad>'])\n",
        "loss_func.cuda(gpu)\n",
        "test_model = ViT_transformer(num_channels = CFG.num_channels,\n",
        "                             pretrained = CFG.pretrained,\n",
        "                              src_dim = 49,\n",
        "                              token_dim = CFG.token_dim,\n",
        "                              decoder_layers = CFG.decoder_layers,\n",
        "                              decoder_heads = CFG.decoder_heads,\n",
        "                              decoder_one_head_dim = CFG.decoder_one_head_dim,\n",
        "                              decoder_ff_dim = CFG.decoder_ff_dim,\n",
        "                              decoder_dropout = CFG.dropout,\n",
        "                              vocab_length = CFG.vocab_length,\n",
        "                              tgt_embedding_dropout = CFG.dropout,\n",
        "                              tgt_max = CFG.max_text_len)\n",
        "test_model.cuda(gpu)\n",
        "\n",
        "optimizer = torch.optim.Adam(\n",
        "    test_model.parameters(), lr = 0.5, betas = (0.9, 0.98), eps = 1e-9\n",
        ")\n",
        "\n",
        "lr_scheduler = LambdaLR(\n",
        "    optimizer = optimizer,\n",
        "    lr_lambda = lambda step: utils.lr_scheduler(\n",
        "        step, model_size= test_model.tgt_embedding.token_dim, factor = 1.0, warmup_step = 400\n",
        "    ),\n",
        ")\n",
        "\n",
        "for epoch in range(CFG.num_epochs):\n",
        "    valid_df = full_df.loc[full_df['fold'] == epoch]\n",
        "    train_df = full_df.loc[full_df['fold'] != epoch]\n",
        "    ds = MolecularTranslationDataModule(train_df, valid_df, tokenizer, CFG, get_aug(CFG))\n",
        "    test_model.train()\n",
        "    print(f\"Now Training Epoch {epoch} ====\", flush = True)\n",
        "    _, train_state = run_one_epoch(\n",
        "      (utils.Batchify(b[0], b[1], tokenizer.text2index['<pad>'], CFG.device) for b in ds.train_dataloader()),\n",
        "      test_model,\n",
        "      loss_func,\n",
        "      optimizer,\n",
        "      lr_scheduler,\n",
        "      CFG,\n",
        "      tokenizer,\n",
        "      mode = \"train\",\n",
        "      train_state = Train_Counter(),\n",
        "      start_idx = tokenizer.text2index['<sos>']\n",
        "    )\n",
        "    GPUtil.showUtilization()\n",
        "    torch.cuda.empty_cache()\n",
        "\n",
        "    test_model.eval()\n",
        "    with torch.no_grad():\n",
        "      print(f\"Now Validating Epoch {epoch} ====\", flush = True)\n",
        "      sloss = run_one_epoch(\n",
        "          (utils.Batchify(b[0], b[1], tokenizer.text2index['<pad>'], CFG.device) for b in ds.val_dataloader()),\n",
        "          test_model,\n",
        "          loss_func,\n",
        "          DummyOptimizer(),\n",
        "          DummyScheduler(),\n",
        "          CFG,\n",
        "          tokenizer,\n",
        "          mode= 'validation',\n",
        "          train_state = Train_Counter(),\n",
        "          start_idx = tokenizer.text2index['<sos>']\n",
        "      )\n",
        "      print(\"Validation Loss {}\".format(sloss))\n",
        "    torch.cuda.empty_cache()\n",
        "torch.save(test_model.state_dict(), \"InChItrainslationfinalmodel.pt\")"
      ],
      "id": "714c8627-2309-44c0-aec5-9e9df6b91e37"
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "_BAF2BHrhTzB"
      },
      "outputs": [],
      "source": [],
      "id": "_BAF2BHrhTzB"
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "machine_shape": "hm",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.9.12"
    },
    "widgets": {
      "application/vnd.jupyter.widget-state+json": {
        "002c6b943d3e4c2d9ea90103e43b640f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ec9d78130bbe44d6beecf320290a5334",
              "IPY_MODEL_3bb599248e0946aa97ae0f82f5b22563",
              "IPY_MODEL_95601eca7f4842d7a57d9c7f789c0f72"
            ],
            "layout": "IPY_MODEL_8c8da9cd6039442597b78ddf29d49062"
          }
        },
        "052869ac78eb4f90a82b065a8d959512": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "056839ede048437fa9a967491af87b61": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_8be6e60b1f374dc5a06e8bd54d259e1a",
              "IPY_MODEL_3569ca4f10194535bbb3a51a571a6b97",
              "IPY_MODEL_96a8848677874f52a62832ffbba399a9"
            ],
            "layout": "IPY_MODEL_bee1f223ee044ebf9b23e7c44d14ea74"
          }
        },
        "08ec8ac1bdc74f2dad32da40e7f9b9ca": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "0d2553205e6e4cad9de36e037bfbc2a7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_f0ca9fc0ef6343cdace5066aa00e1146",
            "max": 100001,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_5572794fb9594b93b04cef0c1ffe6227",
            "value": 100001
          }
        },
        "10eea632aae845e1851798d2a578f679": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "225e62dc323942bb829ff99971b2865f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "22f464229025430cb061b22757ee8fa7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_db4e8503746a473d9cec26e954b639fd",
            "max": 100001,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_99382da64dfa48bb8acf59bbd83fb5c4",
            "value": 100001
          }
        },
        "2765c90a1c214c91aa6aa9f258751186": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "279dd157e57f4eb29c5f1ef86d9986d0": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_a1ded267a2df4c35bd3d3a441f2e098d",
              "IPY_MODEL_22f464229025430cb061b22757ee8fa7",
              "IPY_MODEL_e6188d8108e94c65ab511d3fc78c7817"
            ],
            "layout": "IPY_MODEL_4b5ac3482ff74e498810a51b6c8db883"
          }
        },
        "3569ca4f10194535bbb3a51a571a6b97": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_c468246a7da344c79db736cb0f04766c",
            "max": 100001,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_d31d9201ab0b4308b31fe179632e4c59",
            "value": 100001
          }
        },
        "37bac2d8c32f462c96475685a4387410": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_96c8a4334695410ab5c032f4f5c8b6b4",
            "max": 100001,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_225e62dc323942bb829ff99971b2865f",
            "value": 100001
          }
        },
        "391eb440cc1b4bdeafa9917af59a6a6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5db97629efc04050a09e49b0ba478e20",
            "placeholder": "​",
            "style": "IPY_MODEL_8e6b5feca4774f77a2a7a855e5eb13e7",
            "value": " 100001/100001 [00:02&lt;00:00, 41724.27it/s]"
          }
        },
        "39456e84676a4d4da9fc0877509d1fbe": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "3bb599248e0946aa97ae0f82f5b22563": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_39456e84676a4d4da9fc0877509d1fbe",
            "max": 100001,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_4317a765371d4f549d30bba469090b2d",
            "value": 100001
          }
        },
        "4317a765371d4f549d30bba469090b2d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "4587faae235f4c02824b51e4ec7c3303": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4591a557c0854e039202a136e6570e3f": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4b5ac3482ff74e498810a51b6c8db883": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "4f942c7ffd4a44e8937bd8e5216a764f": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "5159d0ed18ec4ef0b5aa78ad894f8cf7": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5207f97ed5494a26a44a0390b79e3150": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_2765c90a1c214c91aa6aa9f258751186",
            "placeholder": "​",
            "style": "IPY_MODEL_ab7484b541084ee7abb1e5defff3bf81",
            "value": "100%"
          }
        },
        "5262f01b7b4440dda0c84d4252cd51f9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "5572794fb9594b93b04cef0c1ffe6227": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "5db97629efc04050a09e49b0ba478e20": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "62d43a8be8634433a64268be873d565a": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "65f9e12b2f3a4406880071c2d2f286c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_f42f81aad71c440eb18aba4d5b1f0605",
              "IPY_MODEL_0d2553205e6e4cad9de36e037bfbc2a7",
              "IPY_MODEL_ee7dc8f2a2eb4e00a2cff375554e096c"
            ],
            "layout": "IPY_MODEL_8adf4a14e5bd44ffb4736718c717fd53"
          }
        },
        "664b7aa4c30848018e63b23f7df5bea2": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "FloatProgressModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "FloatProgressModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "ProgressView",
            "bar_style": "success",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5262f01b7b4440dda0c84d4252cd51f9",
            "max": 100001,
            "min": 0,
            "orientation": "horizontal",
            "style": "IPY_MODEL_b4423f39da644553b89f6bdd390a9811",
            "value": 100001
          }
        },
        "695dfad19a134d9fb4daf7842609666c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "71e02a850b1345018b3d6b6c91963600": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "7acdbb3d846d4083a44448219233f5f4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_ddc2ada612d140a7a926d73401d509b9",
              "IPY_MODEL_37bac2d8c32f462c96475685a4387410",
              "IPY_MODEL_391eb440cc1b4bdeafa9917af59a6a6a"
            ],
            "layout": "IPY_MODEL_d15b3a4705424c12816219fe27ef4af1"
          }
        },
        "7d44c263986c413e9d1d2ca7c6568a4c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "87a3676e78434e9a83a112a7a35d57b9": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8adf4a14e5bd44ffb4736718c717fd53": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8be6e60b1f374dc5a06e8bd54d259e1a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_71e02a850b1345018b3d6b6c91963600",
            "placeholder": "​",
            "style": "IPY_MODEL_b384b4c586fc43e09d2de9a657901832",
            "value": "100%"
          }
        },
        "8c8da9cd6039442597b78ddf29d49062": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "8e6b5feca4774f77a2a7a855e5eb13e7": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "95601eca7f4842d7a57d9c7f789c0f72": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_5159d0ed18ec4ef0b5aa78ad894f8cf7",
            "placeholder": "​",
            "style": "IPY_MODEL_cf06f265a52d429f805f13804c7d29a6",
            "value": " 100001/100001 [13:54&lt;00:00, 122.55it/s]"
          }
        },
        "96a8848677874f52a62832ffbba399a9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4587faae235f4c02824b51e4ec7c3303",
            "placeholder": "​",
            "style": "IPY_MODEL_d9715f170e5b4f00a6a1f1a3fbe21bea",
            "value": " 100001/100001 [00:01&lt;00:00, 70911.76it/s]"
          }
        },
        "96c8a4334695410ab5c032f4f5c8b6b4": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "99382da64dfa48bb8acf59bbd83fb5c4": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "9ab8e606e4fe4dc28c655991604d8eba": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b7753d0ace004cb183a257f0aece7799",
            "placeholder": "​",
            "style": "IPY_MODEL_052869ac78eb4f90a82b065a8d959512",
            "value": " 100001/100001 [00:02&lt;00:00, 40991.47it/s]"
          }
        },
        "a1ded267a2df4c35bd3d3a441f2e098d": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_bf8915430fee41ef8355e30835f26754",
            "placeholder": "​",
            "style": "IPY_MODEL_cbf6bd66ae924550945c0efd4efdeeff",
            "value": "100%"
          }
        },
        "ab7484b541084ee7abb1e5defff3bf81": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b384b4c586fc43e09d2de9a657901832": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "b4423f39da644553b89f6bdd390a9811": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "b7753d0ace004cb183a257f0aece7799": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "b99f1f1782924394a0f5802384cd7263": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bbf76c24dcf04344978291d39aa83d6a": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HBoxModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HBoxModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HBoxView",
            "box_style": "",
            "children": [
              "IPY_MODEL_5207f97ed5494a26a44a0390b79e3150",
              "IPY_MODEL_664b7aa4c30848018e63b23f7df5bea2",
              "IPY_MODEL_9ab8e606e4fe4dc28c655991604d8eba"
            ],
            "layout": "IPY_MODEL_87a3676e78434e9a83a112a7a35d57b9"
          }
        },
        "bee1f223ee044ebf9b23e7c44d14ea74": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "bf8915430fee41ef8355e30835f26754": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "c468246a7da344c79db736cb0f04766c": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cad8971c9ff5436ba1c59acdaea81e09": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "cbf6bd66ae924550945c0efd4efdeeff": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "cf06f265a52d429f805f13804c7d29a6": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "d15b3a4705424c12816219fe27ef4af1": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "d31d9201ab0b4308b31fe179632e4c59": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "ProgressStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "ProgressStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "bar_color": null,
            "description_width": ""
          }
        },
        "d9715f170e5b4f00a6a1f1a3fbe21bea": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "db4e8503746a473d9cec26e954b639fd": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "ddc2ada612d140a7a926d73401d509b9": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_62d43a8be8634433a64268be873d565a",
            "placeholder": "​",
            "style": "IPY_MODEL_08ec8ac1bdc74f2dad32da40e7f9b9ca",
            "value": "100%"
          }
        },
        "e52f65136efd4b6b969a9b1718c36e8e": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "DescriptionStyleModel",
          "state": {
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "DescriptionStyleModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "StyleView",
            "description_width": ""
          }
        },
        "e6188d8108e94c65ab511d3fc78c7817": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_10eea632aae845e1851798d2a578f679",
            "placeholder": "​",
            "style": "IPY_MODEL_e52f65136efd4b6b969a9b1718c36e8e",
            "value": " 100001/100001 [00:00&lt;00:00, 559759.20it/s]"
          }
        },
        "ec9d78130bbe44d6beecf320290a5334": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_b99f1f1782924394a0f5802384cd7263",
            "placeholder": "​",
            "style": "IPY_MODEL_7d44c263986c413e9d1d2ca7c6568a4c",
            "value": "100%"
          }
        },
        "ee7dc8f2a2eb4e00a2cff375554e096c": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_cad8971c9ff5436ba1c59acdaea81e09",
            "placeholder": "​",
            "style": "IPY_MODEL_695dfad19a134d9fb4daf7842609666c",
            "value": " 100001/100001 [00:00&lt;00:00, 531352.91it/s]"
          }
        },
        "f0ca9fc0ef6343cdace5066aa00e1146": {
          "model_module": "@jupyter-widgets/base",
          "model_module_version": "1.2.0",
          "model_name": "LayoutModel",
          "state": {
            "_model_module": "@jupyter-widgets/base",
            "_model_module_version": "1.2.0",
            "_model_name": "LayoutModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/base",
            "_view_module_version": "1.2.0",
            "_view_name": "LayoutView",
            "align_content": null,
            "align_items": null,
            "align_self": null,
            "border": null,
            "bottom": null,
            "display": null,
            "flex": null,
            "flex_flow": null,
            "grid_area": null,
            "grid_auto_columns": null,
            "grid_auto_flow": null,
            "grid_auto_rows": null,
            "grid_column": null,
            "grid_gap": null,
            "grid_row": null,
            "grid_template_areas": null,
            "grid_template_columns": null,
            "grid_template_rows": null,
            "height": null,
            "justify_content": null,
            "justify_items": null,
            "left": null,
            "margin": null,
            "max_height": null,
            "max_width": null,
            "min_height": null,
            "min_width": null,
            "object_fit": null,
            "object_position": null,
            "order": null,
            "overflow": null,
            "overflow_x": null,
            "overflow_y": null,
            "padding": null,
            "right": null,
            "top": null,
            "visibility": null,
            "width": null
          }
        },
        "f42f81aad71c440eb18aba4d5b1f0605": {
          "model_module": "@jupyter-widgets/controls",
          "model_module_version": "1.5.0",
          "model_name": "HTMLModel",
          "state": {
            "_dom_classes": [],
            "_model_module": "@jupyter-widgets/controls",
            "_model_module_version": "1.5.0",
            "_model_name": "HTMLModel",
            "_view_count": null,
            "_view_module": "@jupyter-widgets/controls",
            "_view_module_version": "1.5.0",
            "_view_name": "HTMLView",
            "description": "",
            "description_tooltip": null,
            "layout": "IPY_MODEL_4591a557c0854e039202a136e6570e3f",
            "placeholder": "​",
            "style": "IPY_MODEL_4f942c7ffd4a44e8937bd8e5216a764f",
            "value": "100%"
          }
        }
      }
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5
}