_VECTOR_D2V.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# README\n",
    "## Options:\n",
    "### Pretrained model:\n",
    "\n",
    "### Tokenization: How to manage OOV?\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "from nltk.tokenize import TweetTokenizer\n",
    "\n",
    "import copy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "# nltk.download('punkt')\n",
    "# nltk.download('averaged_perceptron_tagger')\n",
    "import re\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "import gensim\n",
    "import gensim.models.word2vec as w2v\n",
    "from gensim.test.utils import common_texts\n",
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_rows', 100)\n",
    "pd.set_option('display.max_columns', None)\n",
    "pd.set_option('display.max_colwidth', 100)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "raw_data = pd.read_csv(\"./data/_PHEME_text.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def vectorize(line):\n",
    "    words = []\n",
    "    for word in line:  # line - iterable, for example list of tokens\n",
    "        try:\n",
    "            w2v_idx = w2v_indices[word]\n",
    "        except KeyError:  # if you does not have a vector for this word in your w2v model, continue\n",
    "            words.append(list(np.zeros(200,)))\n",
    "            continue\n",
    "        words.append(list(w2v_vectors[w2v_idx]))\n",
    "        if not word:\n",
    "            words.append(None)\n",
    "\n",
    "        if len(line) > len(words):\n",
    "            continue\n",
    "    return np.asarray(words)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Tokenization\n",
    "tweet_tokenizer = TweetTokenizer()\n",
    "tweet_tokens = []\n",
    "stop_words = set(stopwords.words('english'))\n",
    "\n",
    "for sent in raw_data.text:\n",
    "    sent = re.sub(r\"http\\S+\", \"&\", sent)\n",
    "    sent = re.sub(r\"@\\S+\", \"@\", sent)\n",
    "    sent = re.sub(r\"#\\S+\", \"#\", sent)\n",
    "    sent = re.sub(r'([^\\s\\w@#&]|_)+', '', sent)\n",
    "    sent = re.sub('', '', sent.lower())\n",
    "    # sent = [tweet_tokenizer.tokenize(sent)]\n",
    "    sent = [tweet_tokenizer.tokenize(sent.lower())]\n",
    "    temp = [token for token in sent[0] if not token in stop_words]\n",
    "    tweet_tokens.append([temp])\n",
    "df_tokens = pd.DataFrame(tweet_tokens, columns=['token'])\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>token</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>[breaking, armed, man, takes, hostage, kosher, grocery, east, paris, &amp;]</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>[#, killers, dead, confirmed, gendarmerie]</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>[top, french, cartoonists, charb, cabu, wolinski, tignous, confirmed, among, dead, #, #, attack,...</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>[police, surrounded, area, #, attack, suspects, believed, &amp;, &amp;]</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>[photo, armed, gunmen, face, police, officers, near, #, hq, paris, &amp;, &amp;]</td>\n    </tr>\n  </tbody>\n</table>\n</div>",
      "text/plain": "                                                                                                 token\n0                              [breaking, armed, man, takes, hostage, kosher, grocery, east, paris, &]\n1                                                           [#, killers, dead, confirmed, gendarmerie]\n2  [top, french, cartoonists, charb, cabu, wolinski, tignous, confirmed, among, dead, #, #, attack,...\n3                                      [police, surrounded, area, #, attack, suspects, believed, &, &]\n4                             [photo, armed, gunmen, face, police, officers, near, #, hq, paris, &, &]"
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tokens.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# DOC2VEC"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "from gensim.models.doc2vec import Doc2Vec, TaggedDocument"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "token\nFalse    5802\ndtype: int64"
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tokens.isna().value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": "(5802, 1)"
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df_tokens.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [],
   "source": [
    "import gensim.downloader as api\n",
    "from gensim.models.word2vec import Word2Vec\n",
    "# corpus = api.load('text8')\n",
    "# wv = api.load('word2vec-google-news-300')\n",
    "# fasttext-wiki-news-subwords-300'\n",
    "#  'glove-twitter-200',\n",
    "model = api.load('glove-twitter-200')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # https://drive.google.com/file/d/0B7XkCwpI5KDYNlNUTTlSS21pQmM/edit\n",
    "# # Load Google Pre trained word2vec model\n",
    "# pretrained_google_news_model = KeyedVectors.load_word2vec_format('input_data/GoogleNews-vectors-negative300.bin', binary=True)\n",
    "\n",
    "# # Access vectors for specific words with a keyed lookup:\n",
    "# vector = pretrained_google_news_model['food']\n",
    "# # see the shape of the vector (300,)\n",
    "# vector.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "w2v_object = model.wv\n",
    "w2v_vectors = w2v_object.vectors # here you load vectors for each word in your model\n",
    "w2v_indices = {word: w2v_object.vocab[word].index for word in w2v_object.vocab} # here you load indices - with whom you can find an index of the particular word in your model "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tagged_data = [TaggedDocument(d, [i]) for i, d in enumerate(tokenized_doc)]\n",
    "tagged_data\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "rosetta",
   "language": "python",
   "name": "rosetta"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  },
  "orig_nbformat": 2
 },
 "nbformat": 4,
 "nbformat_minor": 2
}