Youtube_Huggingface_Transformers_in_Pytorch.ipynb

{
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/PradipNichite/Youtube-Tutorials/blob/main/Youtube_Huggingface_Transformers_in_Pytorch.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "3TcK_MWNUcyh"
      },
      "source": [
        "**Agenda:**\n",
        "\n",
        "1.   What is Transfer Learning.\n",
        "2.   Using Transformes model in Pytorch\n",
        "\n",
        "\n",
        "\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "vpn6NAsLPvyL"
      },
      "source": [
        "**Resources:**\n",
        "2.   https://www.notion.so/Transfer-Learning-Huggingface-Transformers-in-Pytorch-6e8f4c0b7dc34f13a01ec78a02aff9f5\n",
        "\n",
        "\n"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 1,
      "metadata": {
        "id": "OWymbqNSTosA"
      },
      "outputs": [],
      "source": [
        "!pip install transformers -q"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**BERT Embeding**"
      ],
      "metadata": {
        "id": "c_O4rhgNMhL5"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "KaGO6aX7Z8ht",
        "outputId": "8c329d00-a34c-4609-8928-e1997a17d4e1"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_layer_norm.weight', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BaseModelOutput(last_hidden_state=tensor([[[-0.3349, -0.1658, -0.0110,  ..., -0.2789,  0.0950,  0.4630],\n",
              "         [-0.0397, -0.0176, -0.3552,  ..., -0.2689,  0.1300,  0.5941],\n",
              "         [-0.0757,  0.3424,  0.0660,  ..., -0.5298, -0.2513,  0.1726],\n",
              "         [ 0.0395, -0.0122,  0.2089,  ..., -0.3721,  0.1763,  0.1254],\n",
              "         [-0.2027, -0.2934, -0.1556,  ..., -0.5440, -0.0524, -0.0880],\n",
              "         [ 0.9069,  0.1240, -0.5695,  ...,  0.0775, -0.7602, -0.2034]]],\n",
              "       grad_fn=<NativeLayerNormBackward0>), hidden_states=None, attentions=None)"
            ]
          },
          "metadata": {},
          "execution_count": 2
        }
      ],
      "source": [
        "from transformers import DistilBertTokenizer, DistilBertModel\n",
        "tokenizer = DistilBertTokenizer.from_pretrained('distilbert-base-uncased')\n",
        "model = DistilBertModel.from_pretrained(\"distilbert-base-uncased\")\n",
        "text = \"some text for model\"\n",
        "encoded_input = tokenizer(text, return_tensors='pt')\n",
        "output = model(**encoded_input)\n",
        "output"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oLwpCu3VL0r2",
        "outputId": "4072fbe8-8aa1-40e6-91d9-e3c088745030"
      },
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "DistilBertModel(\n",
              "  (embeddings): Embeddings(\n",
              "    (word_embeddings): Embedding(30522, 768, padding_idx=0)\n",
              "    (position_embeddings): Embedding(512, 768)\n",
              "    (LayerNorm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "  )\n",
              "  (transformer): Transformer(\n",
              "    (layer): ModuleList(\n",
              "      (0): TransformerBlock(\n",
              "        (attention): MultiHeadSelfAttention(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "        )\n",
              "        (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (ffn): FFN(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (activation): GELUActivation()\n",
              "        )\n",
              "        (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      )\n",
              "      (1): TransformerBlock(\n",
              "        (attention): MultiHeadSelfAttention(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "        )\n",
              "        (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (ffn): FFN(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (activation): GELUActivation()\n",
              "        )\n",
              "        (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      )\n",
              "      (2): TransformerBlock(\n",
              "        (attention): MultiHeadSelfAttention(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "        )\n",
              "        (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (ffn): FFN(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (activation): GELUActivation()\n",
              "        )\n",
              "        (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      )\n",
              "      (3): TransformerBlock(\n",
              "        (attention): MultiHeadSelfAttention(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "        )\n",
              "        (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (ffn): FFN(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (activation): GELUActivation()\n",
              "        )\n",
              "        (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      )\n",
              "      (4): TransformerBlock(\n",
              "        (attention): MultiHeadSelfAttention(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "        )\n",
              "        (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (ffn): FFN(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (activation): GELUActivation()\n",
              "        )\n",
              "        (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      )\n",
              "      (5): TransformerBlock(\n",
              "        (attention): MultiHeadSelfAttention(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (q_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (k_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (v_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "          (out_lin): Linear(in_features=768, out_features=768, bias=True)\n",
              "        )\n",
              "        (sa_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "        (ffn): FFN(\n",
              "          (dropout): Dropout(p=0.1, inplace=False)\n",
              "          (lin1): Linear(in_features=768, out_features=3072, bias=True)\n",
              "          (lin2): Linear(in_features=3072, out_features=768, bias=True)\n",
              "          (activation): GELUActivation()\n",
              "        )\n",
              "        (output_layer_norm): LayerNorm((768,), eps=1e-12, elementwise_affine=True)\n",
              "      )\n",
              "    )\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "output.last_hidden_state.shape"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "_ZFKDo8yJMbg",
        "outputId": "c5e0205d-d970-45d4-aa33-99935319c31a"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "torch.Size([1, 6, 768])"
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "encoded_input"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "nOta7gPwIy3U",
        "outputId": "1bbca927-de39-44a9-ceee-79f8e1c7ef56"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'input_ids': tensor([[ 101, 2070, 3793, 2005, 2944,  102]]), 'attention_mask': tensor([[1, 1, 1, 1, 1, 1]])}"
            ]
          },
          "metadata": {},
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Auto Classes**"
      ],
      "metadata": {
        "id": "TOY1jqgUMpoV"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "from transformers import AutoTokenizer, AutoModel\n",
        "model = AutoModel.from_pretrained(\"distilbert-base-uncased\")\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"distilbert-base-uncased\")\n",
        "text = \"some text for model\"\n",
        "encoded_input = tokenizer(text, return_tensors='pt')\n",
        "output = model(**encoded_input)\n",
        "output"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "3e3kN6CaB4uL",
        "outputId": "9092232a-ee84-408d-8cfb-7e22c60dda90"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "Some weights of the model checkpoint at distilbert-base-uncased were not used when initializing DistilBertModel: ['vocab_layer_norm.weight', 'vocab_transform.bias', 'vocab_layer_norm.bias', 'vocab_projector.weight', 'vocab_transform.weight', 'vocab_projector.bias']\n",
            "- This IS expected if you are initializing DistilBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
            "- This IS NOT expected if you are initializing DistilBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "BaseModelOutput(last_hidden_state=tensor([[[-0.3349, -0.1658, -0.0110,  ..., -0.2789,  0.0950,  0.4630],\n",
              "         [-0.0397, -0.0176, -0.3552,  ..., -0.2689,  0.1300,  0.5941],\n",
              "         [-0.0757,  0.3424,  0.0660,  ..., -0.5298, -0.2513,  0.1726],\n",
              "         [ 0.0395, -0.0122,  0.2089,  ..., -0.3721,  0.1763,  0.1254],\n",
              "         [-0.2027, -0.2934, -0.1556,  ..., -0.5440, -0.0524, -0.0880],\n",
              "         [ 0.9069,  0.1240, -0.5695,  ...,  0.0775, -0.7602, -0.2034]]],\n",
              "       grad_fn=<NativeLayerNormBackward0>), hidden_states=None, attentions=None)"
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Classification Head**"
      ],
      "metadata": {
        "id": "Z839E4uBMsvG"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 7,
      "metadata": {
        "id": "xXBpMQ4Z32PJ"
      },
      "outputs": [],
      "source": [
        "from transformers import AutoTokenizer, AutoModelForSequenceClassification\n",
        "\n",
        "tokenizer = AutoTokenizer.from_pretrained(\"cardiffnlp/twitter-roberta-base-sentiment\")\n",
        "\n",
        "model = AutoModelForSequenceClassification.from_pretrained(\"cardiffnlp/twitter-roberta-base-sentiment\")"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "model"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ASAWVpJ6uUNq",
        "outputId": "f3b4974e-05a5-4284-a41e-23b77aa68ea9"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "RobertaForSequenceClassification(\n",
              "  (roberta): RobertaModel(\n",
              "    (embeddings): RobertaEmbeddings(\n",
              "      (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
              "      (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
              "      (token_type_embeddings): Embedding(1, 768)\n",
              "      (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "      (dropout): Dropout(p=0.1, inplace=False)\n",
              "    )\n",
              "    (encoder): RobertaEncoder(\n",
              "      (layer): ModuleList(\n",
              "        (0): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (1): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (2): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (3): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (4): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (5): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (6): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (7): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (8): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (9): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (10): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "        (11): RobertaLayer(\n",
              "          (attention): RobertaAttention(\n",
              "            (self): RobertaSelfAttention(\n",
              "              (query): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (key): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (value): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "            (output): RobertaSelfOutput(\n",
              "              (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "              (dropout): Dropout(p=0.1, inplace=False)\n",
              "            )\n",
              "          )\n",
              "          (intermediate): RobertaIntermediate(\n",
              "            (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
              "            (intermediate_act_fn): GELUActivation()\n",
              "          )\n",
              "          (output): RobertaOutput(\n",
              "            (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
              "            (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
              "            (dropout): Dropout(p=0.1, inplace=False)\n",
              "          )\n",
              "        )\n",
              "      )\n",
              "    )\n",
              "  )\n",
              "  (classifier): RobertaClassificationHead(\n",
              "    (dense): Linear(in_features=768, out_features=768, bias=True)\n",
              "    (dropout): Dropout(p=0.1, inplace=False)\n",
              "    (out_proj): Linear(in_features=768, out_features=3, bias=True)\n",
              "  )\n",
              ")"
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 11,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "YAzuK_vc4PJi",
        "outputId": "768f0c00-14ee-48ac-8d8e-d3d825ef13d6"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "SequenceClassifierOutput(loss=None, logits=tensor([[ 2.9619, -0.3124, -2.6414]], grad_fn=<AddmmBackward0>), hidden_states=None, attentions=None)"
            ]
          },
          "metadata": {},
          "execution_count": 11
        }
      ],
      "source": [
        "text = \"I enjoyed movie\"\n",
        "text = \"Movie was waste of time\"\n",
        "encoded_input = tokenizer(text, return_tensors='pt')\n",
        "output = model(**encoded_input)\n",
        "output"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Post Processing**"
      ],
      "metadata": {
        "id": "vyKnqR4JNRp6"
      }
    },
    {
      "cell_type": "code",
      "execution_count": 12,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "5z2CGpzB4S3v",
        "outputId": "3f7dd172-1d70-41ec-bef2-717c441233f4"
      },
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0"
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ],
      "source": [
        "import numpy as np\n",
        "logits = output.logits.detach().numpy() \n",
        "y_pred = np.argmax(logits)\n",
        "y_pred"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": 13,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "dHU4qI3U4ccA",
        "outputId": "54aac180-579c-4262-d0ed-bc95c01a7443"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[0.960126519203186, 0.036334723234176636, 0.0035387093666940928]]\n"
          ]
        }
      ],
      "source": [
        "# Normalize score\n",
        "print(output.logits.softmax(dim=-1).tolist())"
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Prediction on GPU**"
      ],
      "metadata": {
        "id": "FaFdxKHPMzR8"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = model.to('cuda')\n",
        "text = \"I enjoyed movie\"\n",
        "# text = \"Movie was waste of time\"\n",
        "encoded_input = tokenizer(text, return_tensors='pt').to('cuda')\n",
        "output = model(**encoded_input)\n",
        "logits = output.logits.detach().cpu().numpy() \n",
        "print(logits)\n",
        "y_pred = np.argmax(logits)\n",
        "y_pred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "04nQhT2dvXkP",
        "outputId": "a47f54c9-c780-4e89-8582-274c9ae7acdf"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[-2.3957675  -0.23097068  3.1936426 ]]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "2"
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "markdown",
      "source": [
        "**Predicting Multiple Examples**"
      ],
      "metadata": {
        "id": "ifDJ31ikM8cl"
      }
    },
    {
      "cell_type": "code",
      "source": [
        "model = model.to('cuda')\n",
        "examples = [\"I enjoyed movie\",\"Movie was waste of time\"]\n",
        "encoded_input = tokenizer(examples, return_tensors='pt',padding=True,truncation=True).to('cuda')\n",
        "output = model(**encoded_input)\n",
        "logits = output.logits.detach().cpu().numpy() \n",
        "print(logits)\n",
        "\n",
        "y_pred = np.argmax(logits,axis=-1)\n",
        "y_pred"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "axyYaMWfF-Dc",
        "outputId": "61d1619c-9b38-401a-a049-4c16781b990f"
      },
      "execution_count": 19,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[[-2.3957667  -0.23097058  3.1936426 ]\n",
            " [ 2.9619248  -0.31236637 -2.6413777 ]]\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "array([2, 0])"
            ]
          },
          "metadata": {},
          "execution_count": 19
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "# https://www.sharpsightlabs.com/blog/numpy-argmax/"
      ],
      "metadata": {
        "id": "Sr0jwt_eGdFM"
      },
      "execution_count": null,
      "outputs": []
    }
  ],
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "collapsed_sections": [],
      "provenance": [],
      "authorship_tag": "ABX9TyMSmvGQDAV4t8zAS+BmSIfl",
      "include_colab_link": true
    },
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "nbformat": 4,
  "nbformat_minor": 0
}