Class_5_J001.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "name": "Class_5_J001.ipynb",
      "provenance": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/saurabh4006/NLP/blob/master/Class_5_J001.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "e14C_biKvMtH",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 298
        },
        "outputId": "1572cbc1-e2fc-4685-fe5d-d0ab0f13fb9e"
      },
      "source": [
        "!pip install word2vec"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Collecting word2vec\n",
            "\u001b[?25l  Downloading https://files.pythonhosted.org/packages/ce/51/5e2782b204015c8aef0ac830297c2f2735143ec90f592b9b3b909bb89757/word2vec-0.10.2.tar.gz (60kB)\n",
            "\r\u001b[K     |█████▍                          | 10kB 18.3MB/s eta 0:00:01\r\u001b[K     |██████████▉                     | 20kB 1.4MB/s eta 0:00:01\r\u001b[K     |████████████████▎               | 30kB 2.0MB/s eta 0:00:01\r\u001b[K     |█████████████████████▋          | 40kB 1.3MB/s eta 0:00:01\r\u001b[K     |███████████████████████████     | 51kB 1.6MB/s eta 0:00:01\r\u001b[K     |████████████████████████████████| 61kB 1.6MB/s \n",
            "\u001b[?25hRequirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from word2vec) (1.12.0)\n",
            "Requirement already satisfied: cython in /usr/local/lib/python3.6/dist-packages (from word2vec) (0.29.14)\n",
            "Requirement already satisfied: numpy>=1.9.2 in /usr/local/lib/python3.6/dist-packages (from word2vec) (1.17.4)\n",
            "Requirement already satisfied: scipy in /usr/local/lib/python3.6/dist-packages (from word2vec) (1.3.3)\n",
            "Requirement already satisfied: scikit-learn in /usr/local/lib/python3.6/dist-packages (from word2vec) (0.21.3)\n",
            "Requirement already satisfied: joblib>=0.11 in /usr/local/lib/python3.6/dist-packages (from scikit-learn->word2vec) (0.14.1)\n",
            "Building wheels for collected packages: word2vec\n",
            "  Building wheel for word2vec (setup.py) ... \u001b[?25l\u001b[?25hdone\n",
            "  Created wheel for word2vec: filename=word2vec-0.10.2-cp36-cp36m-linux_x86_64.whl size=128665 sha256=ff5b7a09def3f7af7a5d96252aa7cea9313d3a607212ebb1b617c6f2bbce0fab\n",
            "  Stored in directory: /root/.cache/pip/wheels/ef/9f/06/aec42532c9c37e05f936d4d586b15cfdfc9f2ffb62bd7fed1c\n",
            "Successfully built word2vec\n",
            "Installing collected packages: word2vec\n",
            "Successfully installed word2vec-0.10.2\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "QqDO7ZQYvO-U",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "import pandas as pd\n",
        "import numpy as np\n",
        "from gensim.models import Word2Vec\n",
        "from nltk.corpus import stopwords\n",
        "from itertools import combinations\n",
        "import string"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nepqMkbMvV3o",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "data=pd.read_csv(\"https://raw.githubusercontent.com/zfz/twitter_corpus/master/full-corpus.csv\")\n",
        "twitter = data.copy()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "GuZJ_7Nwvbsh",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "twitter['tokenText'] = twitter['TweetText'].apply(lambda x: list(set(x.translate(str.maketrans('','', string.punctuation)).split())))"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "wOnGpkFfvkeQ",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 228
        },
        "outputId": "1da1640b-47bd-41c7-8788-953a23eddf21"
      },
      "source": [
        "twitter['tokenText']"
      ],
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "0       [that, be, on, get, has, do, will, to, swype, ...\n",
              "1       [adding, be, carrier, will, to, Apple, more, t...\n",
              "2       [video, with, Siri, does, httptco8ExbnQjY, sum...\n",
              "3       [made, too, iPhone, me, you, it, to, RIM, Appl...\n",
              "4       [was, that, into, I, ios5, the, realized, twit...\n",
              "                              ...                        \n",
              "5108                         [me, re, copè, twitter, con]\n",
              "5109    [los, Buenas, noches, twitter, quierooo, gente...\n",
              "5110    [costumbre, tiene, me, yo, ponerce, mala, cuan...\n",
              "5111    [profissional, Muito, followback, Sempre, Perm...\n",
              "5112    [tudo, JesusLucas, foii, a, os, e, twitter, pa...\n",
              "Name: tokenText, Length: 5113, dtype: object"
            ]
          },
          "metadata": {
            "tags": []
          },
          "execution_count": 5
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "Qazrdx65vqd-",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "words = [[i] for i in list(set([y for x in twitter['tokenText'] for y in x]))]\n",
        "model = Word2Vec(list(words), min_count=1,size= 50,workers=3, window =3, sg = 1)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "R6TUDXqXvxVk",
        "colab_type": "code",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 864
        },
        "outputId": "06bbaf10-169f-4a14-f3de-0254a284ee71"
      },
      "source": [
        "for i in words[0:10]:\n",
        "  for j in words[0:10]:\n",
        "    if j>i:\n",
        "      print(i[0],j[0],' : ',model.wv.similarity(i[0],j[0]))\n",
        "    else:\n",
        "      continue"
      ],
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "MsAmberPRiley testify  :  0.4337415\n",
            "MsAmberPRiley “Like”  :  0.045621462\n",
            "MsAmberPRiley iAm  :  -0.12289686\n",
            "MsAmberPRiley texts  :  0.21768564\n",
            "MsAmberPRiley expectations  :  0.20260315\n",
            "MsAmberPRiley bricked  :  0.03611538\n",
            "MsAmberPRiley utilisé  :  0.02059264\n",
            "MsAmberPRiley “Playful  :  0.19190249\n",
            "MsAmberPRiley بالمر  :  0.027919644\n",
            "testify “Like”  :  0.013140276\n",
            "testify texts  :  -0.0037233636\n",
            "testify utilisé  :  0.061392598\n",
            "testify “Playful  :  0.08132692\n",
            "testify بالمر  :  -0.098633\n",
            "“Like” “Playful  :  0.16394731\n",
            "iAm testify  :  -0.29457366\n",
            "iAm “Like”  :  0.17481707\n",
            "iAm texts  :  -0.10177477\n",
            "iAm utilisé  :  0.1451509\n",
            "iAm “Playful  :  0.16058728\n",
            "iAm بالمر  :  0.3770713\n",
            "texts “Like”  :  0.20785646\n",
            "texts utilisé  :  -0.09484253\n",
            "texts “Playful  :  0.0714892\n",
            "texts بالمر  :  0.21368882\n",
            "expectations testify  :  0.15594591\n",
            "expectations “Like”  :  0.05771197\n",
            "expectations iAm  :  -0.03137225\n",
            "expectations texts  :  0.024795095\n",
            "expectations utilisé  :  0.07832217\n",
            "expectations “Playful  :  -0.10151959\n",
            "expectations بالمر  :  0.14080869\n",
            "bricked testify  :  0.058092147\n",
            "bricked “Like”  :  0.020847445\n",
            "bricked iAm  :  -0.05201578\n",
            "bricked texts  :  -0.045588046\n",
            "bricked expectations  :  0.037270755\n",
            "bricked utilisé  :  0.05277131\n",
            "bricked “Playful  :  -0.1428061\n",
            "bricked بالمر  :  -0.21715318\n",
            "utilisé “Like”  :  -0.27426603\n",
            "utilisé “Playful  :  -0.0969102\n",
            "utilisé بالمر  :  0.08408301\n",
            "بالمر “Like”  :  0.11358679\n",
            "بالمر “Playful  :  0.16309908\n"
          ],
          "name": "stdout"
        },
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/gensim/matutils.py:737: FutureWarning: Conversion of the second argument of issubdtype from `int` to `np.signedinteger` is deprecated. In future, it will be treated as `np.int64 == np.dtype(int).type`.\n",
            "  if np.issubdtype(vec.dtype, np.int):\n"
          ],
          "name": "stderr"
        }
      ]
    }
  ]
}