Sentence_Similarity_using_T5base.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "language": "python",
      "name": "python3"
    },
    "language_info": {
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "file_extension": ".py",
      "mimetype": "text/x-python",
      "name": "python",
      "nbconvert_exporter": "python",
      "pygments_lexer": "ipython3",
      "version": "3.8.5"
    },
    "colab": {
      "name": "Sentence Similarity using BERT.ipynb",
      "provenance": []
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3ItWaD1oHlJs",
        "outputId": "d550bb92-c5d3-40e9-fb83-8d8de8a2b91b"
      },
      "source": [
        "!pip install sentencepiece\n",
        "!pip install transformers"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/14/67/e42bd1181472c95c8cda79305df848264f2a7f62740995a46945d9797b67/sentencepiece-0.1.95-cp36-cp36m-manylinux2014_x86_64.whl (1.2MB)\n",
            "\r\u001b[K     |▎                               | 10kB 5.3MB/s eta 0:00:01\r\u001b[K     |▌                               | 20kB 7.1MB/s eta 0:00:01\r\u001b[K     |▉                               | 30kB 6.4MB/s eta 0:00:01\r\u001b[K     |█                               | 40kB 4.7MB/s eta 0:00:01\r\u001b[K     |█▍                              | 51kB 5.8MB/s eta 0:00:01\r\u001b[K     |█▋                              | 61kB 6.8MB/s eta 0:00:01\r\u001b[K     |██                              | 71kB 7.3MB/s eta 0:00:01\r\u001b[K     |██▏                             | 81kB 7.6MB/s eta 0:00:01\r\u001b[K     |██▌                             | 92kB 8.0MB/s eta 0:00:01\r\u001b[K     |██▊                             | 102kB 6.6MB/s eta 0:00:01\r\u001b[K     |███                             | 112kB 6.6MB/s eta 0:00:01\r\u001b[K     |███▎                            | 122kB 6.6MB/s eta 0:00:01\r\u001b[K     |███▌                            | 133kB 6.6MB/s eta 0:00:01\r\u001b[K     |███▉                            | 143kB 6.6MB/s eta 0:00:01\r\u001b[K     |████                            | 153kB 6.6MB/s eta 0:00:01\r\u001b[K     |████▍                           | 163kB 6.6MB/s eta 0:00:01\r\u001b[K     |████▋                           | 174kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████                           | 184kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████▏                          | 194kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████▌                          | 204kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████▊                          | 215kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████                          | 225kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████▎                         | 235kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████▌                         | 245kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████▉                         | 256kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████                         | 266kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████▍                        | 276kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████▋                        | 286kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████                        | 296kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████▏                       | 307kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████▍                       | 317kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████▊                       | 327kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████                       | 337kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████▎                      | 348kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████▌                      | 358kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████▉                      | 368kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████                      | 378kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████▍                     | 389kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████▋                     | 399kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████                     | 409kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████▏                    | 419kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████▍                    | 430kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████▊                    | 440kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████                    | 450kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████▎                   | 460kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████▌                   | 471kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████▉                   | 481kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████                   | 491kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████▍                  | 501kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████▋                  | 512kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████▉                  | 522kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████▏                 | 532kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████▍                 | 542kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████▊                 | 552kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████                 | 563kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████▎                | 573kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████▌                | 583kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████▉                | 593kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████                | 604kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████▍               | 614kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████▋               | 624kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████▉               | 634kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▏              | 645kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▍              | 655kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████▊              | 665kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████              | 675kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▎             | 686kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▌             | 696kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████▉             | 706kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████             | 716kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▎            | 727kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▋            | 737kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████▉            | 747kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▏           | 757kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▍           | 768kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████▊           | 778kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████           | 788kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▎          | 798kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▌          | 808kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████▉          | 819kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████          | 829kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▎         | 839kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▋         | 849kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████▉         | 860kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▏        | 870kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▍        | 880kB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████▊        | 890kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████        | 901kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▎       | 911kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▌       | 921kB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████▊       | 931kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████       | 942kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▎      | 952kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▋      | 962kB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████▉      | 972kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▏     | 983kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▍     | 993kB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████▊     | 1.0MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 1.0MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▎    | 1.0MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▌    | 1.0MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████▊    | 1.0MB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████    | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▎   | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▋   | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████▉   | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▏  | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▍  | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |█████████████████████████████▊  | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████  | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▏ | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▌ | 1.1MB 6.6MB/s eta 0:00:01\r\u001b[K     |██████████████████████████████▊ | 1.2MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████ | 1.2MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▎| 1.2MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▋| 1.2MB 6.6MB/s eta 0:00:01\r\u001b[K     |███████████████████████████████▉| 1.2MB 6.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 1.2MB 6.6MB/s \n",
            "\u001b[?25hInstalling collected packages: sentencepiece\n",
            "Successfully installed sentencepiece-0.1.95\n",
            "Collecting transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/0c/7d5950fcd80b029be0a8891727ba21e0cd27692c407c51261c3c921f6da3/transformers-4.1.1-py3-none-any.whl (1.5MB)\n",
            "\u001b[K     |████████████████████████████████| 1.5MB 7.5MB/s \n",
            "\u001b[?25hRequirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Collecting tokenizers==0.9.4\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/0f/1c/e789a8b12e28be5bc1ce2156cf87cb522b379be9cadc7ad8091a4cc107c4/tokenizers-0.9.4-cp36-cp36m-manylinux2010_x86_64.whl (2.9MB)\n",
            "\u001b[K     |████████████████████████████████| 2.9MB 29.7MB/s \n",
            "\u001b[?25hRequirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.19.5)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.8)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/7d/34/09d19aff26edcc8eb2a01bed8e98f13a1537005d31e95233fd48216eed10/sacremoses-0.0.43.tar.gz (883kB)\n",
            "\u001b[K     |████████████████████████████████| 890kB 41.0MB/s \n",
            "\u001b[?25hRequirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.12.5)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.15.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.0.0)\n",
            "Building wheels for collected packages: sacremoses\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.43-cp36-none-any.whl size=893261 sha256=55ed63dbfa968891ba20a950ba36d144923da5c575f720fdae7adefec89342d1\n",
            "  Stored in directory: /root/.cache/pip/wheels/29/3c/fd/7ce5c3f0666dab31a50123635e6fb5e19ceb42ce38d4e58f45\n",
            "Successfully built sacremoses\n",
            "Installing collected packages: tokenizers, sacremoses, transformers\n",
            "Successfully installed sacremoses-0.0.43 tokenizers-0.9.4 transformers-4.1.1\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "XVqTZ-M26QZF"
      },
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import tensorflow as tf\n",
        "import transformers"
      ],
      "execution_count": 2,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "bgaxObWi6QZH"
      },
      "source": [
        "## Setting configuration"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QRVF3Gvt6QZH"
      },
      "source": [
        "max_length = 128\n",
        "batch_size = 32\n",
        "epochs = 2\n",
        "# The labels in our dataset\n",
        "labels = [\"contradition\",\"entailment\",\"neutral\"]"
      ],
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "6QpqiaRN6QZH"
      },
      "source": [
        "## Load the data"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Sj-DFsiB6QZI",
        "outputId": "abd221fb-c7e3-44fb-d5b3-bf850ae22ebf"
      },
      "source": [
        "!curl -LO https://raw.githubusercontent.com/MohamadMerchant/SNLI/master/data.tar.gz\n",
        "!tar -xvzf data.tar.gz"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "  % Total    % Received % Xferd  Average Speed   Time    Time     Time  Current\n",
            "                                 Dload  Upload   Total   Spent    Left  Speed\n",
            "100 11.1M  100 11.1M    0     0  18.9M      0 --:--:-- --:--:-- --:--:-- 18.8M\n",
            "SNLI_Corpus/\n",
            "SNLI_Corpus/snli_1.0_dev.csv\n",
            "SNLI_Corpus/snli_1.0_train.csv\n",
            "SNLI_Corpus/snli_1.0_test.csv\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "QxayzR2H6QZI",
        "outputId": "7b9d4c4a-810f-4c54-f1e7-b4e567803d70"
      },
      "source": [
        "train_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_train.csv\",nrows=100000)\n",
        "valid_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_dev.csv\")\n",
        "test_df = pd.read_csv(\"SNLI_Corpus/snli_1.0_test.csv\")\n",
        "\n",
        "\n",
        "print(\"Total train samples : {}\".format(train_df.shape[0]))\n",
        "print(\"Total validionat samples : {}\".format(valid_df.shape[0]))\n",
        "print(\"Total train samples : {}\".format(train_df.shape[0]))"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Total train samples : 100000\n",
            "Total validionat samples : 10000\n",
            "Total train samples : 100000\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 204
        },
        "id": "H1oInn5g6QZJ",
        "outputId": "4137c747-2e9a-46e3-9af5-85aca45da5ab"
      },
      "source": [
        "train_df.head()"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>neutral</td>\n",
              "      <td>A person on a horse jumps over a broken down a...</td>\n",
              "      <td>A person is training his horse for a competition.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>A person on a horse jumps over a broken down a...</td>\n",
              "      <td>A person is at a diner, ordering an omelette.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>entailment</td>\n",
              "      <td>A person on a horse jumps over a broken down a...</td>\n",
              "      <td>A person is outdoors, on a horse.</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>neutral</td>\n",
              "      <td>Children smiling and waving at camera</td>\n",
              "      <td>They are smiling at their parents</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>entailment</td>\n",
              "      <td>Children smiling and waving at camera</td>\n",
              "      <td>There are children present</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ...                                          sentence2\n",
              "0        neutral  ...  A person is training his horse for a competition.\n",
              "1  contradiction  ...      A person is at a diner, ordering an omelette.\n",
              "2     entailment  ...                  A person is outdoors, on a horse.\n",
              "3        neutral  ...                  They are smiling at their parents\n",
              "4     entailment  ...                         There are children present\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "7g79LcYJ6QZK"
      },
      "source": [
        "There are three similarity values in the dataset\n",
        "\n",
        "**neutral** the sentences are neutral\n",
        "\n",
        "**contradiction** the sentances are not similar\n",
        "\n",
        "**entailment** the sentances have similar meanings\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "IXWVK5nj6QZK"
      },
      "source": [
        "## Preprocessing"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "i7aJXwbo6QZK",
        "outputId": "1ad2f9e1-dd97-4635-e22f-989d7761b96a"
      },
      "source": [
        "print(\"Number of missing values = \\n {}\".format(train_df.isnull().sum()))"
      ],
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Number of missing values = \n",
            " similarity    0\n",
            "sentence1     0\n",
            "sentence2     3\n",
            "dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k516krsG6QZL"
      },
      "source": [
        "train_df.dropna(axis=0,inplace=True)"
      ],
      "execution_count": 8,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "tiW5O9uE6QZL",
        "outputId": "b9bc3d75-398b-46d7-a3b0-e8c82e4b513b"
      },
      "source": [
        "print(\"Train Target Distribution\")\n",
        "print(train_df.similarity.value_counts())"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Train Target Distribution\n",
            "entailment       33384\n",
            "contradiction    33310\n",
            "neutral          33193\n",
            "-                  110\n",
            "Name: similarity, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2M7k_tH16QZL",
        "outputId": "43662a39-e715-44ee-e30a-457a8d8136d7"
      },
      "source": [
        "print(\"Validation Target Distribution\")\n",
        "print(valid_df.similarity.value_counts())"
      ],
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Validation Target Distribution\n",
            "entailment       3329\n",
            "contradiction    3278\n",
            "neutral          3235\n",
            "-                 158\n",
            "Name: similarity, dtype: int64\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "mUF5Vs6Y6QZL"
      },
      "source": [
        "The value \"-\" is appreading in the distribution so we need to remove that"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JbRPYj7a6QZM"
      },
      "source": [
        "train_df = (train_df[train_df.similarity!=\"-\"]\n",
        "            .sample(frac=1.0,random_state=42)\n",
        "            .reset_index(drop=True)\n",
        "           )\n",
        "valid_df = (valid_df[valid_df.similarity!=\"-\"]\n",
        "            .sample(frac=1.0,random_state=42)\n",
        "            .reset_index(drop=True)\n",
        "           )"
      ],
      "execution_count": 11,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "4ApLs3Bb6QZM"
      },
      "source": [
        "One hot encoding the vectors"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "cwQdeSg46QZM"
      },
      "source": [
        "train_df[\"label\"] = train_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_train = tf.keras.utils.to_categorical(train_df.label, num_classes=3)\n",
        "\n",
        "valid_df[\"label\"] = valid_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_val = tf.keras.utils.to_categorical(valid_df.label, num_classes=3)\n",
        "\n",
        "test_df[\"label\"] = test_df[\"similarity\"].apply(\n",
        "    lambda x: 0 if x == \"contradiction\" else 1 if x == \"entailment\" else 2\n",
        ")\n",
        "y_test = tf.keras.utils.to_categorical(test_df.label, num_classes=3)"
      ],
      "execution_count": 12,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 272
        },
        "id": "KECsDAdu6QZM",
        "outputId": "b26caa73-61db-4972-ae04-529027dabaaa"
      },
      "source": [
        "train_df.head()"
      ],
      "execution_count": 13,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>similarity</th>\n",
              "      <th>sentence1</th>\n",
              "      <th>sentence2</th>\n",
              "      <th>label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>Two male clowns, one in a plaid suit and the o...</td>\n",
              "      <td>The clowns are in the dressing room.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>A man is playing music with a trumpet.</td>\n",
              "      <td>A man playing music with a guitar.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>neutral</td>\n",
              "      <td>A man with a hard hat carries a tree branch.</td>\n",
              "      <td>The man with the hard hat had cut down a tree.</td>\n",
              "      <td>2</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>entailment</td>\n",
              "      <td>An old bearded man holding a fruit.</td>\n",
              "      <td>An elderly bearded man holding a piece of frui...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>contradiction</td>\n",
              "      <td>A man in a blue coat looks at his feet in a ba...</td>\n",
              "      <td>A man is looking up at the ceiling in a grocer...</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "      similarity  ... label\n",
              "0  contradiction  ...     0\n",
              "1  contradiction  ...     0\n",
              "2        neutral  ...     2\n",
              "3     entailment  ...     1\n",
              "4  contradiction  ...     0\n",
              "\n",
              "[5 rows x 4 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 13
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "HD7Hg-w76QZN",
        "outputId": "4ff859ed-585b-483c-e841-eb3a6e89f908"
      },
      "source": [
        "y_train"
      ],
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([[1., 0., 0.],\n",
              "       [1., 0., 0.],\n",
              "       [0., 0., 1.],\n",
              "       ...,\n",
              "       [1., 0., 0.],\n",
              "       [1., 0., 0.],\n",
              "       [0., 0., 1.]], dtype=float32)"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "lmGleTCo6QZN"
      },
      "source": [
        "## Custom keras data generator"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "xx3gWWeA6QZN"
      },
      "source": [
        "class t5SemanticDataGenerator(tf.keras.utils.Sequence):\n",
        "\n",
        "\n",
        "    def __init__(\n",
        "        self,\n",
        "        sentence_pairs,\n",
        "        labels,\n",
        "        batch_size=batch_size,\n",
        "        shuffle=True,\n",
        "        include_targets=True,\n",
        "    ):\n",
        "        self.sentence_pairs = sentence_pairs\n",
        "        self.labels = labels\n",
        "        self.shuffle = shuffle\n",
        "        self.batch_size = batch_size\n",
        "        self.include_targets = include_targets\n",
        "\n",
        "        self.tokenizer = transformers.T5Tokenizer.from_pretrained(\n",
        "            \"t5-base\", do_lower_case=True\n",
        "        )\n",
        "        self.indexes = np.arange(len(self.sentence_pairs))\n",
        "        self.on_epoch_end()\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.sentence_pairs) // self.batch_size\n",
        "\n",
        "    def __getitem__(self, idx):\n",
        "        indexes = self.indexes[idx * self.batch_size : (idx + 1) * self.batch_size]\n",
        "        sentence_pairs = self.sentence_pairs[indexes]\n",
        "\n",
        "\n",
        "        encoded = self.tokenizer.batch_encode_plus(\n",
        "            sentence_pairs.tolist(),\n",
        "            add_special_tokens=True,\n",
        "            max_length=max_length,\n",
        "            return_attention_mask=True,\n",
        "            return_token_type_ids=True,\n",
        "            pad_to_max_length=True,\n",
        "            return_tensors=\"tf\",\n",
        "        )\n",
        "\n",
        "        input_ids = np.array(encoded[\"input_ids\"], dtype=\"int32\")\n",
        "        attention_masks = np.array(encoded[\"attention_mask\"], dtype=\"int32\")\n",
        "\n",
        "        if self.include_targets:\n",
        "            labels = np.array(self.labels[indexes], dtype=\"int32\")\n",
        "            return [input_ids, attention_masks], labels\n",
        "        else:\n",
        "            return [input_ids, attention_masks]\n",
        "\n",
        "    def on_epoch_end(self):\n",
        "        if self.shuffle:\n",
        "            np.random.RandomState(42).shuffle(self.indexes)"
      ],
      "execution_count": 27,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "Y2ztcr1R6QZO"
      },
      "source": [
        "## Build the model"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "px2-cmTS6QZO",
        "outputId": "71f6bcf5-c6aa-43e9-f71a-10a38a107c8f"
      },
      "source": [
        "strategy = tf.distribute.MirroredStrategy()\n",
        "\n",
        "with strategy.scope():\n",
        "    input_ids = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32, name=\"input_ids\"\n",
        "    )\n",
        "    attention_masks = tf.keras.layers.Input(\n",
        "        shape=(max_length,), dtype=tf.int32,name=\"attention_masks\"\n",
        "    )\n",
        "  \n",
        "  \n",
        "    t5_model = transformers.TFT5Model.from_pretrained(\"t5-base\")\n",
        "    \n",
        "    t5_model.trainable = False\n",
        "    \n",
        "    output = t5_model(\n",
        "        input_ids,attention_mask=attention_masks,decoder_input_ids=input_ids\n",
        "    )\n",
        "    sequence_output, pooled_output = output[:2]\n",
        "    \n",
        "    bi_lstm = tf.keras.layers.Bidirectional(\n",
        "        tf.keras.layers.LSTM(64, return_sequences=True)\n",
        "    )(sequence_output)\n",
        "    avg_pool = tf.keras.layers.GlobalAveragePooling1D()(bi_lstm)\n",
        "    max_pool = tf.keras.layers.GlobalMaxPooling1D()(bi_lstm)\n",
        "    concat = tf.keras.layers.concatenate([avg_pool,max_pool])\n",
        "    dropout = tf.keras.layers.Dropout(0.3)(concat)\n",
        "    output = tf.keras.layers.Dense(3,activation=\"softmax\")(dropout)\n",
        "    model = tf.keras.models.Model(\n",
        "        inputs=[input_ids,attention_masks],outputs=output\n",
        "    )\n",
        "    \n",
        "    model.compile(\n",
        "        optimizer=tf.keras.optimizers.Adam(),\n",
        "        loss=\"categorical_crossentropy\",\n",
        "        metrics=[\"acc\"],\n",
        "    )\n",
        "    \n",
        "print(f\"Strategy: {strategy}\")\n",
        "model.summary()"
      ],
      "execution_count": 28,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "INFO:tensorflow:Using MirroredStrategy with devices ('/job:localhost/replica:0/task:0/device:GPU:0',)\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "All model checkpoint layers were used when initializing TFT5Model.\n",
            "\n",
            "All the layers of TFT5Model were initialized from the model checkpoint at t5-base.\n",
            "If your task is similar to the task the model of the checkpoint was trained on, you can already use TFT5Model for predictions without further training.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Strategy: <tensorflow.python.distribute.mirrored_strategy.MirroredStrategy object at 0x7f1da3609f98>\n",
            "Model: \"model_3\"\n",
            "__________________________________________________________________________________________________\n",
            "Layer (type)                    Output Shape         Param #     Connected to                     \n",
            "==================================================================================================\n",
            "input_ids (InputLayer)          [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "attention_masks (InputLayer)    [(None, 128)]        0                                            \n",
            "__________________________________________________________________________________________________\n",
            "tf_t5model_3 (TFT5Model)        TFSeq2SeqModelOutput 222903552   input_ids[0][0]                  \n",
            "                                                                 attention_masks[0][0]            \n",
            "                                                                 input_ids[0][0]                  \n",
            "__________________________________________________________________________________________________\n",
            "bidirectional_3 (Bidirectional) (None, 128, 128)     426496      tf_t5model_3[0][1]               \n",
            "__________________________________________________________________________________________________\n",
            "global_average_pooling1d_3 (Glo (None, 128)          0           bidirectional_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "global_max_pooling1d_3 (GlobalM (None, 128)          0           bidirectional_3[0][0]            \n",
            "__________________________________________________________________________________________________\n",
            "concatenate_3 (Concatenate)     (None, 256)          0           global_average_pooling1d_3[0][0] \n",
            "                                                                 global_max_pooling1d_3[0][0]     \n",
            "__________________________________________________________________________________________________\n",
            "dropout_491 (Dropout)           (None, 256)          0           concatenate_3[0][0]              \n",
            "__________________________________________________________________________________________________\n",
            "dense_3 (Dense)                 (None, 3)            771         dropout_491[0][0]                \n",
            "==================================================================================================\n",
            "Total params: 223,330,819\n",
            "Trainable params: 427,267\n",
            "Non-trainable params: 222,903,552\n",
            "__________________________________________________________________________________________________\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "EhK-KmJD6QZO"
      },
      "source": [
        "train_data = t5SemanticDataGenerator(\n",
        "    train_df[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_train,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=True,\n",
        ")\n",
        "valid_data = t5SemanticDataGenerator(\n",
        "    valid_df[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_val,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=False,\n",
        ")"
      ],
      "execution_count": 29,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "eb_RWJMx6QZP",
        "outputId": "45805897-793a-45a7-e1c8-1bd94866def2"
      },
      "source": [
        "history = model.fit(\n",
        "    train_data,\n",
        "    validation_data=valid_data,\n",
        "    epochs=epochs,\n",
        "    use_multiprocessing=True,\n",
        "    workers=-1,\n",
        ")"
      ],
      "execution_count": 30,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Epoch 1/2\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "3121/3121 [==============================] - ETA: 0s - loss: 1.0242 - acc: 0.4669"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\b\r3121/3121 [==============================] - 2199s 695ms/step - loss: 1.0242 - acc: 0.4669 - val_loss: 0.7693 - val_acc: 0.6496\n",
            "Epoch 2/2\n",
            "3121/3121 [==============================] - 2167s 694ms/step - loss: 0.8495 - acc: 0.6128 - val_loss: 0.6954 - val_acc: 0.7065\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "VVravxDu6QZP"
      },
      "source": [
        "# Fine-tuning"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "oMWMopff6QZQ"
      },
      "source": [
        "# Evaluate model on the test set"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "cINytjy66QZQ",
        "outputId": "8f531c1a-4228-4711-8e92-5dd66233b8c4"
      },
      "source": [
        "test_data = t5SemanticDataGenerator(\n",
        "    test_df[[\"sentence1\", \"sentence2\"]].values.astype(\"str\"),\n",
        "    y_test,\n",
        "    batch_size=batch_size,\n",
        "    shuffle=False,\n",
        ")\n",
        "model.evaluate(test_data, verbose=1)"
      ],
      "execution_count": 34,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "312/312 [==============================] - 172s 529ms/step - loss: 0.7109 - accuracy: 0.6971\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[0.7109248042106628, 0.6971153616905212]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 34
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "a6i_HVIi6QZQ"
      },
      "source": [
        "# Inference on custom sentences"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "_E5gt95B6QZQ"
      },
      "source": [
        "def check_similarity(sentence1, sentence2):\n",
        "    sentence_pairs = np.array([[str(sentence1), str(sentence2)]])\n",
        "    test_data = t5SemanticDataGenerator(\n",
        "        sentence_pairs, labels=None, batch_size=1, shuffle=False, include_targets=False,\n",
        "    )\n",
        "\n",
        "    proba = model.predict(test_data)[0]\n",
        "    idx = np.argmax(proba)\n",
        "    proba = f\"{proba[idx]: .2f}%\"\n",
        "    pred = labels[idx]\n",
        "    return pred, proba"
      ],
      "execution_count": 39,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "Fat7ZDVp6QZQ",
        "outputId": "0431c405-bb34-41f9-c370-9084db370782"
      },
      "source": [
        "sentence1 = \"Two women are observing something together.\"\n",
        "sentence2 = \"Two women are standing with their eyes closed.\"\n",
        "check_similarity(sentence1, sentence2)"
      ],
      "execution_count": 40,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n",
            "The parameters `output_attentions`, `output_hidden_states` and `use_cache` cannot be updated when calling a model.They have to be set to True/False in the config object (i.e.: `config=XConfig.from_pretrained('name', output_attentions=True)`).\n",
            "The parameter `return_dict` cannot be set in graph mode and will always be set to `True`.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('contradition', ' 0.72%')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 40
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "FA8zoiJ-6QZR",
        "outputId": "8bc59742-5102-45d6-a87c-0ae5908f4c0d"
      },
      "source": [
        "sentence1 = \"A smiling costumed woman is holding an umbrella\"\n",
        "sentence2 = \"A happy woman in a fairy costume holds an umbrella\"\n",
        "check_similarity(sentence1, sentence2)"
      ],
      "execution_count": 41,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('neutral', ' 0.50%')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 41
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "SZN2KCjS6QZR",
        "outputId": "ffd3441f-87e1-43ec-8acb-4da33846602f"
      },
      "source": [
        "sentence1 = \"A soccer game with multiple males playing\"\n",
        "sentence2 = \"Some men are playing a sport\"\n",
        "check_similarity(sentence1, sentence2)"
      ],
      "execution_count": 42,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Truncation was not explicitly activated but `max_length` is provided a specific value, please use `truncation=True` to explicitly truncate examples to max length. Defaulting to 'longest_first' truncation strategy. If you encode pairs of sequences (GLUE-style) with the tokenizer you can select this strategy more precisely by providing a specific strategy to `truncation`.\n",
            "/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2179: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).\n",
            "  FutureWarning,\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "('entailment', ' 0.68%')"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 42
        }
      ]
    }
  ]
}