BERT_sentiment_Analysis.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Latest_Stanford_SentoimentAnal.ipynb",
      "provenance": [],
      "collapsed_sections": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "accelerator": "GPU"
  },
  "cells": [
    {
      "cell_type": "code",
      "metadata": {
        "id": "39_7k4vJSFio",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 384
        },
        "outputId": "999d4660-9b5a-4909-be83-00cfa4303a58"
      },
      "source": [
        "!pip install transformers\n",
        "!pip install wget"
      ],
      "execution_count": 167,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Requirement already satisfied: transformers in /usr/local/lib/python3.6/dist-packages (3.0.2)\n",
            "Requirement already satisfied: requests in /usr/local/lib/python3.6/dist-packages (from transformers) (2.23.0)\n",
            "Requirement already satisfied: filelock in /usr/local/lib/python3.6/dist-packages (from transformers) (3.0.12)\n",
            "Requirement already satisfied: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers) (2019.12.20)\n",
            "Requirement already satisfied: sentencepiece!=0.1.92 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.1.91)\n",
            "Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from transformers) (1.18.5)\n",
            "Requirement already satisfied: tokenizers==0.8.1.rc1 in /usr/local/lib/python3.6/dist-packages (from transformers) (0.8.1rc1)\n",
            "Requirement already satisfied: sacremoses in /usr/local/lib/python3.6/dist-packages (from transformers) (0.0.43)\n",
            "Requirement already satisfied: dataclasses; python_version < \"3.7\" in /usr/local/lib/python3.6/dist-packages (from transformers) (0.7)\n",
            "Requirement already satisfied: tqdm>=4.27 in /usr/local/lib/python3.6/dist-packages (from transformers) (4.41.1)\n",
            "Requirement already satisfied: packaging in /usr/local/lib/python3.6/dist-packages (from transformers) (20.4)\n",
            "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2.10)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (2020.6.20)\n",
            "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (1.24.3)\n",
            "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers) (3.0.4)\n",
            "Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (1.12.0)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (0.16.0)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers) (7.1.2)\n",
            "Requirement already satisfied: pyparsing>=2.0.2 in /usr/local/lib/python3.6/dist-packages (from packaging->transformers) (2.4.7)\n",
            "Requirement already satisfied: wget in /usr/local/lib/python3.6/dist-packages (3.2)\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "9iBqu3He1tSD",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "\n",
        "from os import path\n",
        "\n",
        "inputfile=\"aclImdb_v1.tar.gz\"\n",
        "\n",
        "if not path.exists(inputfile) :\n",
        "  !wget https://ai.stanford.edu/~amaas/data/sentiment/aclImdb_v1.tar.gz\n",
        "  !tar xvfz aclImdb_v1.tar.gz"
      ],
      "execution_count": 168,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gLSjueAc2nZa",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#!ls aclImdb/test/neg/*"
      ],
      "execution_count": 169,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0SM0MWJgBoCu",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "ce1876ae-7965-46de-b7e7-f13a505d8b7d"
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/gdrive')"
      ],
      "execution_count": 170,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/gdrive; to attempt to forcibly remount, call drive.mount(\"/content/gdrive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Xrx2yKjfSbDi",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import os\n",
        "import torch\n",
        "import torch.nn as nn\n",
        "import transformers\n",
        "from transformers import BertForSequenceClassification,AdamW,BertTokenizer\n",
        "from transformers import get_linear_schedule_with_warmup\n",
        "from torchsummary import summary\n",
        "\n",
        "\n",
        "from tqdm.auto import tqdm \n",
        "import time\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "from sklearn import metrics\n",
        "from sklearn import model_selection\n",
        "from sklearn.preprocessing import OneHotEncoder\n"
      ],
      "execution_count": 171,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Dr3-3_8s_cJy",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "MAX_LEN=512\n",
        "TRAIN_BATCH_SIZE = 4\n",
        "VALID_BATCH_SIZE = 4\n",
        "EPOCHS = 3\n",
        "MODEL_PATH=\"/content/gdrive/My Drive/bert_stanford_sent_anal_model.bin\"\n",
        "OUTPUT_LOG=\"/content/gdrive/My Drive/bert_stanford_sent_anal_train.log\""
      ],
      "execution_count": 172,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "WEHun85ZlQki",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "9c46cca8-6828-43ca-f1ff-3d6b90175d25"
      },
      "source": [
        "if torch.cuda.is_available() :\n",
        "  device = torch.device(\"cuda\")\n",
        "  print('We will use the GPU:',torch.cuda.get_device_name(0))\n",
        "else:\n",
        "  print('No GPU available, using the CPU instead')\n",
        "  device = torch.device(\"cpu\")"
      ],
      "execution_count": 173,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "We will use the GPU: Tesla T4\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "YWRGCIvGlKOg",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 35
        },
        "outputId": "00862ee6-512d-4157-f6e2-f57932a5bac2"
      },
      "source": [
        "# Load the BERT tokenizer.\n",
        "print('Loading BERT tokenizer...')\n",
        "tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)\n",
        "enc = OneHotEncoder(handle_unknown='ignore')"
      ],
      "execution_count": 174,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Loading BERT tokenizer...\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bo3WLsqKWbUJ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def log(value):\n",
        "  if os.path.exists(OUTPUT_LOG) :\n",
        "    f= open(OUTPUT_LOG,\"a\")\n",
        "    f.write(value+\"\\n\")\n",
        "    f.close()\n",
        "  else:\n",
        "    f= open(OUTPUT_LOG,\"w\")\n",
        "    f.write(value+ \"\\n\")\n",
        "    f.close()\n"
      ],
      "execution_count": 175,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "FBBVaXWI4HUV",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Model\n",
        "class BERTBaseUncased(nn.Module):\n",
        "    def __init__(self):\n",
        "        super(BERTBaseUncased, self).__init__()\n",
        "        self.bert = BertForSequenceClassification.from_pretrained(\n",
        "                      \"bert-base-uncased\", # Use the 12-layer BERT model, with an uncased vocab.\n",
        "                      #num_labels = 2, # The number of output labels--2 for binary classification.\n",
        "                      #output_attentions = False, # Whether the model returns attentions weights.\n",
        "                      #output_hidden_states = False, # Whether the model returns all hidden-states.\n",
        "                      )\n",
        "    def forward(self, ids, mask, token_type_ids):\n",
        "        o2 = self.bert(ids, attention_mask=mask, token_type_ids=token_type_ids)\n",
        "        return o2[0]"
      ],
      "execution_count": 176,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bRPgA9NfouD9",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 176,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "UsneONZY6cdI",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "class BERTDataset:\n",
        "    def __init__(self, input_text, target):\n",
        "        self.input_text = input_text\n",
        "        self.target = target\n",
        "        self.tokenizer = tokenizer\n",
        "        self.max_len = MAX_LEN\n",
        "\n",
        "    def __len__(self):\n",
        "        return len(self.input_text)\n",
        "\n",
        "    def __getitem__(self, item):\n",
        "        input_text = str(self.input_text[item])\n",
        "        input_text = \" \".join(input_text.split())\n",
        "        \n",
        "        inputs = self.tokenizer.encode_plus(\n",
        "            input_text,\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=self.max_len,\n",
        "            pad_to_max_length=True,\n",
        "            truncation='longest_first'\n",
        "        )\n",
        "        \n",
        "        ids = inputs[\"input_ids\"]\n",
        "        mask = inputs[\"attention_mask\"]\n",
        "        token_type_ids = inputs[\"token_type_ids\"]\n",
        "\n",
        "        return {\n",
        "            \"ids\": torch.tensor(ids, dtype=torch.long),\n",
        "            \"mask\": torch.tensor(mask, dtype=torch.long),\n",
        "            \"token_type_ids\": torch.tensor(token_type_ids, dtype=torch.long),\n",
        "            \"targets\": torch.tensor(self.target[item], dtype=torch.float),\n",
        "        }\n"
      ],
      "execution_count": 177,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e_flTAfsEPXM",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 177,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "98cgTC6E4tun",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Engine\n",
        "\n",
        "def loss_fn(outputs, targets):\n",
        "    #print(\"OUTPUT\",outputs)\n",
        "    return nn.BCEWithLogitsLoss()(outputs, targets)\n",
        "\n",
        "\n",
        "def train_fn(data_loader, model, optimizer, device, scheduler):\n",
        "    model.train()\n",
        "\n",
        "    for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):\n",
        "        \n",
        "        ids = d[\"ids\"]\n",
        "        token_type_ids = d[\"token_type_ids\"]\n",
        "        mask = d[\"mask\"]\n",
        "        targets = d[\"targets\"]\n",
        "\n",
        "        #print(\"ids\",ids.shape)\n",
        "        #print(\"attn\",mask.shape)\n",
        "        #print(\"token\",token_type_ids.shape)\n",
        "\n",
        "        ids = ids.to(device, dtype=torch.long)\n",
        "        token_type_ids = token_type_ids.to(device, dtype=torch.long)\n",
        "        mask = mask.to(device, dtype=torch.long)\n",
        "        targets = targets.to(device, dtype=torch.float)\n",
        "\n",
        "        optimizer.zero_grad()\n",
        "        outputs = model(ids, mask, token_type_ids=token_type_ids)\n",
        "\n",
        "        loss = loss_fn(outputs, targets)\n",
        "        loss.backward()\n",
        "        optimizer.step()\n",
        "        scheduler.step()\n",
        "        \n",
        "\n",
        "\n",
        "def eval_fn(data_loader, model, device):\n",
        "    model.eval()\n",
        "    fin_targets = []\n",
        "    fin_outputs = []\n",
        "    with torch.no_grad():\n",
        "        for bi, d in tqdm(enumerate(data_loader), total=len(data_loader)):\n",
        "            ids = d[\"ids\"]\n",
        "            token_type_ids = d[\"token_type_ids\"]\n",
        "            mask = d[\"mask\"]\n",
        "            targets = d[\"targets\"]\n",
        "\n",
        "            ids = ids.to(device, dtype=torch.long)\n",
        "            token_type_ids = token_type_ids.to(device, dtype=torch.long)\n",
        "            mask = mask.to(device, dtype=torch.long)\n",
        "            targets = targets.to(device, dtype=torch.float)\n",
        "\n",
        "            outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)\n",
        "            fin_targets.extend(targets.cpu().detach().numpy().tolist())\n",
        "            fin_outputs.extend(torch.sigmoid(outputs).cpu().detach().numpy().tolist())\n",
        "    return fin_outputs, fin_targets\n"
      ],
      "execution_count": 178,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "a3z0eevOt2DA",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def read_data(dir_path):\n",
        "  classes=['pos','neg']\n",
        "  values=[]\n",
        "  targets=[]\n",
        "  for cls in classes:\n",
        "    base_path=dir_path + \"/\" + cls\n",
        "    for entry in os.listdir(base_path):\n",
        "      filename=os.path.join(base_path, entry)\n",
        "      if os.path.isfile(filename):\n",
        "        with open(filename, 'r') as file:\n",
        "          data = file.read()\n",
        "          values.append(data)\n",
        "          targets.append(cls)\n",
        "  #enc.fit(targets.reshape(-1,1))\n",
        "  \n",
        "  targets=enc.fit_transform(np.array(targets).reshape(-1,1)).toarray()\n",
        "  print(targets.shape)\n",
        "  return values,targets\n"
      ],
      "execution_count": 179,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "3SzNSCebRNyZ",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def show_tokens(sentence):\n",
        "    max_len = MAX_LEN\n",
        "    input_text = str(sentence)\n",
        "    input_text = \" \".join(input_text.split())\n",
        "\n",
        "  \n",
        "    inputs = tokenizer.encode_plus(\n",
        "            input_text,\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=max_len,\n",
        "            pad_to_max_length=True,\n",
        "            truncation='longest_first'\n",
        "        )\n",
        "\n",
        "    ids = inputs[\"input_ids\"]\n",
        "    mask = inputs[\"attention_mask\"]\n",
        "    token_type_ids = inputs[\"token_type_ids\"]\n",
        "    print(ids)\n",
        "    print(mask)\n",
        "    print(token_type_ids)\n",
        "\n",
        "#show_tokens(\"Nice  picturization of the song on the beaches of Tahiti.[SEP]Very good acting by Abhishek and Aishwarya. \")\n",
        "#show_tokens(\"Good Movie\")"
      ],
      "execution_count": 180,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "JOoe4Fgv8ds8",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Training\n",
        "\n",
        "import random\n",
        "\n",
        "def train():\n",
        " \n",
        "    train_path='aclImdb/train'\n",
        "    test_path='aclImdb/test'\n",
        "\n",
        "    train_values,train_targets=read_data(train_path)\n",
        "    test_values,test_targets=read_data(test_path)\n",
        "\n",
        "    #for i in range(0,10) :\n",
        "      #print(train_targets[i],train_values[i])\n",
        "\n",
        "    print(len(train_targets))\n",
        "    print(len(train_values))\n",
        "\n",
        "    print(len(test_targets))\n",
        "    print(len(test_values))\n",
        "\n",
        "    choices = list(range(len(train_values)))\n",
        "    random.shuffle(choices)\n",
        "\n",
        "    new_train_values = []\n",
        "    new_train_targets = []\n",
        "\n",
        "    for n in choices :\n",
        "      new_train_values.append(train_values[choices[n]])\n",
        "      new_train_targets.append(train_targets[choices[n]])\n",
        "\n",
        "    train_dataset = BERTDataset(        \n",
        "        input_text=new_train_values, target=new_train_targets\n",
        "    )\n",
        "\n",
        "    train_data_loader = torch.utils.data.DataLoader(\n",
        "        train_dataset, batch_size=TRAIN_BATCH_SIZE, num_workers=4,\n",
        "    )\n",
        "\n",
        "    valid_dataset = BERTDataset(\n",
        "        input_text=test_values, target=test_targets\n",
        "    )\n",
        "\n",
        "    valid_data_loader = torch.utils.data.DataLoader(\n",
        "        valid_dataset, batch_size=VALID_BATCH_SIZE, num_workers=1,\n",
        "    )\n",
        "\n",
        "    log(\"Train Size=\"+str(len(train_dataset)))\n",
        "    log(\"Valid size=\"+str(len(valid_dataset)))\n",
        "    \n",
        "    model = BERTBaseUncased()\n",
        "    model.to(device)\n",
        "    #print(model)\n",
        "    param_optimizer = list(model.named_parameters())\n",
        "    no_decay = [\"bias\", \"LayerNorm.bias\", \"LayerNorm.weight\"]\n",
        "    optimizer_parameters = [\n",
        "        {\n",
        "            \"params\": [\n",
        "                p for n, p in param_optimizer if not any(nd in n for nd in no_decay)\n",
        "            ],\n",
        "            \"weight_decay\": 0.001,\n",
        "        },\n",
        "        {\n",
        "            \"params\": [\n",
        "                p for n, p in param_optimizer if any(nd in n for nd in no_decay)\n",
        "            ],\n",
        "            \"weight_decay\": 0.0,\n",
        "        },\n",
        "    ]\n",
        "\n",
        "    num_train_steps = int(len(train_targets) / TRAIN_BATCH_SIZE * EPOCHS)\n",
        "    optimizer = AdamW(optimizer_parameters, lr=3e-5)\n",
        "    scheduler = get_linear_schedule_with_warmup(\n",
        "        optimizer, num_warmup_steps=0, num_training_steps=num_train_steps\n",
        "    )\n",
        "\n",
        "    model = nn.DataParallel(model)\n",
        "\n",
        "    best_accuracy = 0\n",
        "    for epoch in range(EPOCHS):\n",
        "        train_fn(train_data_loader, model, optimizer, device, scheduler)\n",
        "        print(\"Calling eval\")\n",
        "        outputs, targets = eval_fn(valid_data_loader, model, device)\n",
        "        outputs = np.array(outputs) >= 0.5\n",
        "        accuracy = metrics.accuracy_score(targets, outputs)\n",
        "        log(\" Epoch=\"+str(epoch)+\"Accuracy=\"+str(accuracy))\n",
        "        print(f\"Accuracy Score = {accuracy}\")\n",
        "        if accuracy > best_accuracy:\n",
        "            model_to_save = model.module if hasattr(model, 'module') else model\n",
        "            torch.save(model_to_save.state_dict(), MODEL_PATH)\n",
        "            best_accuracy = accuracy\n",
        "    return model"
      ],
      "execution_count": 181,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZDEL-VdeB_Yk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "#Predict\n",
        "PREDICTION_CACHE = dict()\n",
        "\n",
        "def predict_from_cache(model,sentence):\n",
        "    if sentence in PREDICTION_DICT:\n",
        "        return PREDICTION_DICT[sentence]\n",
        "    else:\n",
        "        result = sentence_prediction(model,sentence)\n",
        "        PREDICTION_CACHE[sentence] = result\n",
        "        return result\n",
        "\n",
        "def sentence_prediction(model,sentence):\n",
        "    max_len = MAX_LEN\n",
        "    input_text = str(sentence)\n",
        "    input_text = \" \".join(input_text.split())\n",
        "\n",
        "  \n",
        "    inputs = tokenizer.encode_plus(\n",
        "            input_text,\n",
        "            None,\n",
        "            add_special_tokens=True,\n",
        "            max_length=max_len,\n",
        "            #pad_to_max_length=True,\n",
        "            truncation='longest_first'\n",
        "        )\n",
        "\n",
        "    ids = inputs[\"input_ids\"]\n",
        "    mask = inputs[\"attention_mask\"]\n",
        "    token_type_ids = inputs[\"token_type_ids\"]\n",
        "\n",
        "    padding_length = max_len - len(ids)\n",
        "    ids = ids + ([0] * padding_length)\n",
        "    mask = mask + ([0] * padding_length)\n",
        "    token_type_ids = token_type_ids + ([0] * padding_length)\n",
        "\n",
        "    ids = torch.tensor(ids, dtype=torch.long).unsqueeze(0)\n",
        "    mask = torch.tensor(mask, dtype=torch.long).unsqueeze(0)\n",
        "    token_type_ids = torch.tensor(token_type_ids, dtype=torch.long).unsqueeze(0)\n",
        "\n",
        "    ids = ids.to(device, dtype=torch.long)\n",
        "    token_type_ids = token_type_ids.to(device, dtype=torch.long)\n",
        "    mask = mask.to(device, dtype=torch.long)\n",
        "    \n",
        "    outputs = model(ids=ids, mask=mask, token_type_ids=token_type_ids)\n",
        "    outputs = torch.sigmoid(outputs[0]).cpu().detach().numpy()\n",
        "    print(outputs)\n",
        "\n",
        "    cls = np.round(outputs.reshape(-1,2))\n",
        "    return enc.inverse_transform(cls)"
      ],
      "execution_count": 185,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "sCvF8cLWDDEB",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "def predict() :\n",
        "  \n",
        "  model=BERTBaseUncased()\n",
        "  model.load_state_dict(torch.load(MODEL_PATH))\n",
        "  model.to(device)\n",
        "    \n",
        "  out = sentence_prediction(model,\"Sholay is a good movie, I like Thanku's acting very much\")\n",
        "  print(out)\n",
        "  out = sentence_prediction(model,\"Good Movie\")\n",
        "  print(out)\n",
        "  out= sentence_prediction(model,\"Uski Roti was a boring movie, there is no drama , just the routine visits to home by husband etc.\")\n",
        "  print(out)\n",
        "  out= sentence_prediction(model,\"bad movie\")\n",
        "  print(out)"
      ],
      "execution_count": 186,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ET2CA8_YPZlk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 263
        },
        "outputId": "0f038c06-6be9-41d2-8dc1-bb47e312f7ca"
      },
      "source": [
        "#train()\n",
        "predict()"
      ],
      "execution_count": 187,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Some weights of the model checkpoint at bert-base-uncased were not used when initializing BertForSequenceClassification: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
            "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
            "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
            "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at bert-base-uncased and are newly initialized: ['classifier.weight', 'classifier.bias']\n",
            "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "[0.04918659 0.9535181 ]\n",
            "[['pos']]\n",
            "[0.06349492 0.94567746]\n",
            "[['pos']]\n",
            "[0.945852   0.05560359]\n",
            "[['neg']]\n",
            "[0.9330367  0.07159735]\n",
            "[['neg']]\n"
          ],
          "name": "stdout"
        }
      ]
    }
  ]
}