Generate_sentence_vectors.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Generate_sentence_vectors.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/daniellehausler/nlp_active_learning/blob/master/Generate_sentence_vectors.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vkLDYu4WpvpA",
        "colab_type": "code",
        "outputId": "ffdc7c41-62f0-408a-a50f-3623a91f72f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 787
        }
      },
      "source": [
        "pip install -U sentence-transformers"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting sentence-transformers\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/c9/91/c85ddef872d5bb39949386930c1f834ac382e145fcd30155b09d6fb65c5a/sentence-transformers-0.2.5.tar.gz (49kB)\n",
            "\u001b[K     |████████████████████████████████| 51kB 1.7MB/s \n",
            "\u001b[?25hCollecting transformers==2.3.0\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/50/10/aeefced99c8a59d828a92cc11d213e2743212d3641c87c82d61b035a7d5c/transformers-2.3.0-py3-none-any.whl (447kB)\n",
            "\u001b[K     |████████████████████████████████| 450kB 7.9MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: tqdm in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (4.28.1)\n",
            "Requirement already satisfied, skipping upgrade: torch>=1.0.1 in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.4.0)\n",
            "Requirement already satisfied, skipping upgrade: numpy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.17.5)\n",
            "Requirement already satisfied, skipping upgrade: scikit-learn in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (0.22.1)\n",
            "Requirement already satisfied, skipping upgrade: scipy in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (1.4.1)\n",
            "Requirement already satisfied, skipping upgrade: nltk in /usr/local/lib/python3.6/dist-packages (from sentence-transformers) (3.2.5)\n",
            "Requirement already satisfied, skipping upgrade: boto3 in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0->sentence-transformers) (1.11.15)\n",
            "Requirement already satisfied, skipping upgrade: requests in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0->sentence-transformers) (2.21.0)\n",
            "Collecting sacremoses\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/a6/b4/7a41d630547a4afd58143597d5a49e07bfd4c42914d8335b2a5657efc14b/sacremoses-0.0.38.tar.gz (860kB)\n",
            "\u001b[K     |████████████████████████████████| 870kB 48.6MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: regex!=2019.12.17 in /usr/local/lib/python3.6/dist-packages (from transformers==2.3.0->sentence-transformers) (2019.12.20)\n",
            "Collecting sentencepiece\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/74/f4/2d5214cbf13d06e7cb2c20d84115ca25b53ea76fa1f0ade0e3c9749de214/sentencepiece-0.1.85-cp36-cp36m-manylinux1_x86_64.whl (1.0MB)\n",
            "\u001b[K     |████████████████████████████████| 1.0MB 42.1MB/s \n",
            "\u001b[?25hRequirement already satisfied, skipping upgrade: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->sentence-transformers) (0.14.1)\n",
            "Requirement already satisfied, skipping upgrade: six in /usr/local/lib/python3.6/dist-packages (from nltk->sentence-transformers) (1.12.0)\n",
            "Requirement already satisfied, skipping upgrade: botocore<1.15.0,>=1.14.15 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.3.0->sentence-transformers) (1.14.15)\n",
            "Requirement already satisfied, skipping upgrade: jmespath<1.0.0,>=0.7.1 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.3.0->sentence-transformers) (0.9.4)\n",
            "Requirement already satisfied, skipping upgrade: s3transfer<0.4.0,>=0.3.0 in /usr/local/lib/python3.6/dist-packages (from boto3->transformers==2.3.0->sentence-transformers) (0.3.3)\n",
            "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.3.0->sentence-transformers) (2019.11.28)\n",
            "Requirement already satisfied, skipping upgrade: urllib3<1.25,>=1.21.1 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.3.0->sentence-transformers) (1.24.3)\n",
            "Requirement already satisfied, skipping upgrade: idna<2.9,>=2.5 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.3.0->sentence-transformers) (2.8)\n",
            "Requirement already satisfied, skipping upgrade: chardet<3.1.0,>=3.0.2 in /usr/local/lib/python3.6/dist-packages (from requests->transformers==2.3.0->sentence-transformers) (3.0.4)\n",
            "Requirement already satisfied, skipping upgrade: click in /usr/local/lib/python3.6/dist-packages (from sacremoses->transformers==2.3.0->sentence-transformers) (7.0)\n",
            "Requirement already satisfied, skipping upgrade: docutils<0.16,>=0.10 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers==2.3.0->sentence-transformers) (0.15.2)\n",
            "Requirement already satisfied, skipping upgrade: python-dateutil<3.0.0,>=2.1 in /usr/local/lib/python3.6/dist-packages (from botocore<1.15.0,>=1.14.15->boto3->transformers==2.3.0->sentence-transformers) (2.6.1)\n",
            "Building wheels for collected packages: sentence-transformers, sacremoses\n",
            "  Building wheel for sentence-transformers (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sentence-transformers: filename=sentence_transformers-0.2.5-cp36-none-any.whl size=64942 sha256=3267768dbba8e49ce8e625dee08e7d3536c5d0ac7f62cf1f294bec61aafd0ff5\n",
            "  Stored in directory: /root/.cache/pip/wheels/b4/ce/39/5bbda8ac34eb52df8c6531382ca077773fbfcbfb6386e5d66c\n",
            "  Building wheel for sacremoses (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for sacremoses: filename=sacremoses-0.0.38-cp36-none-any.whl size=884628 sha256=05d3d0d26e19d77f14ce58aee0760fbbc6f5a073b6cede83ce7ff3dfda5d4390\n",
            "  Stored in directory: /root/.cache/pip/wheels/6d/ec/1a/21b8912e35e02741306f35f66c785f3afe94de754a0eaf1422\n",
            "Successfully built sentence-transformers sacremoses\n",
            "Installing collected packages: sacremoses, sentencepiece, transformers, sentence-transformers\n",
            "Successfully installed sacremoses-0.0.38 sentence-transformers-0.2.5 sentencepiece-0.1.85 transformers-2.3.0\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "GIbtj83YuLNE",
        "colab_type": "text"
      },
      "source": [
        "Import Pre-trained model and initiate it "
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vpGffRzPsLkL",
        "colab_type": "code",
        "outputId": "f0283516-2776-4562-f4e1-0a56a7b17fab",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 81
        }
      },
      "source": [
        "from sentence_transformers import SentenceTransformer\n",
        "model = SentenceTransformer('bert-large-nli-stsb-mean-tokens')"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/html": [
              "<p style=\"color: red;\">\n",
              "The default version of TensorFlow in Colab will soon switch to TensorFlow 2.x.<br>\n",
              "We recommend you <a href=\"https://www.tensorflow.org/guide/migrate\" target=\"_blank\">upgrade</a> now \n",
              "or ensure your notebook will continue to use TensorFlow 1.x via the <code>%tensorflow_version 1.x</code> magic:\n",
              "<a href=\"https://colab.research.google.com/notebooks/tensorflow_version.ipynb\" target=\"_blank\">more info</a>.</p>\n"
            ],
            "text/plain": [
              "<IPython.core.display.HTML object>"
            ]
          },
          "metadata": {
            "tags": []
          }
        },
        {
          "output_type": "stream",
          "text": [
            "100%|██████████| 1.24G/1.24G [00:45<00:00, 27.1MB/s]\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "toFE4Q8ufsdB",
        "colab_type": "code",
        "outputId": "de34cb31-a50f-482d-a768-df6d2c2f20d6",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 203
        }
      },
      "source": [
        "#import files - read txt file into df of raw text and label\n",
        "import pandas as pd\n",
        "file = '/content/yelp_labelled.txt'\n",
        "df = pd.read_table(file,header=0,names=['sentence','Label'])\n",
        "df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>Label</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Crust is not good.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Not tasty and the texture was just nasty.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Stopped by during the late May bank holiday of...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The selection on the menu was great and so wer...</td>\n",
              "      <td>1</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Now I am getting angry and I want my damn pho.</td>\n",
              "      <td>0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            sentence  Label\n",
              "0                                 Crust is not good.      0\n",
              "1          Not tasty and the texture was just nasty.      0\n",
              "2  Stopped by during the late May bank holiday of...      1\n",
              "3  The selection on the menu was great and so wer...      1\n",
              "4     Now I am getting angry and I want my damn pho.      0"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "0XXJPkHWxROH",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "pnvvnky4xSOx",
        "colab_type": "text"
      },
      "source": [
        ""
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vF3-o1CRsNq7",
        "colab_type": "code",
        "outputId": "0b6a8bf3-fa5a-46d0-e4e4-c6b85292713c",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 328
        }
      },
      "source": [
        "short_df = df.head(1000)\n",
        "sentence_list = short_df.sentence.astype(str).tolist()\n",
        "sentence_embeddings = model.encode(sentence_list)\n",
        "short_df['embedded'] = sentence_embeddings\n",
        "short_df.head()"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:4: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  after removing the cwd from sys.path.\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/html": [
              "<div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>sentence</th>\n",
              "      <th>Label</th>\n",
              "      <th>embedded</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>Crust is not good.</td>\n",
              "      <td>0</td>\n",
              "      <td>[0.6439732, 0.056025624, -0.35271582, 0.819160...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>Not tasty and the texture was just nasty.</td>\n",
              "      <td>0</td>\n",
              "      <td>[-0.13985315, 0.416396, 0.4409965, 0.4977425, ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Stopped by during the late May bank holiday of...</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.36495483, -0.24059258, 0.7212093, -0.883463...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>The selection on the menu was great and so wer...</td>\n",
              "      <td>1</td>\n",
              "      <td>[0.5756608, -0.71696526, -0.10919548, 0.042644...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>Now I am getting angry and I want my damn pho.</td>\n",
              "      <td>0</td>\n",
              "      <td>[0.26188856, -0.24982847, 0.33465514, 0.349182...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>"
            ],
            "text/plain": [
              "                                            sentence  ...                                           embedded\n",
              "0                                 Crust is not good.  ...  [0.6439732, 0.056025624, -0.35271582, 0.819160...\n",
              "1          Not tasty and the texture was just nasty.  ...  [-0.13985315, 0.416396, 0.4409965, 0.4977425, ...\n",
              "2  Stopped by during the late May bank holiday of...  ...  [0.36495483, -0.24059258, 0.7212093, -0.883463...\n",
              "3  The selection on the menu was great and so wer...  ...  [0.5756608, -0.71696526, -0.10919548, 0.042644...\n",
              "4     Now I am getting angry and I want my damn pho.  ...  [0.26188856, -0.24982847, 0.33465514, 0.349182...\n",
              "\n",
              "[5 rows x 3 columns]"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RRuPLk3mm3m3",
        "colab_type": "code",
        "outputId": "a6f464e9-73fb-4d52-8ce6-95dd4898c0fc",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 69
        }
      },
      "source": [
        "from scipy.spatial import distance\n",
        "\n",
        "v = short_df.loc[89]\n",
        "u = short_df.loc[20]\n",
        "print(\"u:\",u.sentence)\n",
        "print(\"v:\",v.sentence)\n",
        "distance.cosine(u.embedded,v.embedded)\n"
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "u: The food, amazing.\n",
            "v: In summary, this was a largely disappointing dining experience.\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0.6117225289344788"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 26
        }
      ]
    }
  ]
}