word2vec_model_on_avazon_review.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data loaded\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import, division, print_function\n",
    "import pandas as pd\n",
    "# load the data into panda dataframe\n",
    "data_file_name = \"Health_and_Personal_Care_5.json\"\n",
    "raw_df = pd.read_json(data_file_name, lines=True)\n",
    "print(\"Data loaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "Int64Index: 346355 entries, 0 to 346354\n",
      "Data columns (total 9 columns):\n",
      "asin              346355 non-null object\n",
      "helpful           346355 non-null object\n",
      "overall           346355 non-null int64\n",
      "reviewText        346355 non-null object\n",
      "reviewTime        346355 non-null object\n",
      "reviewerID        346355 non-null object\n",
      "reviewerName      343304 non-null object\n",
      "summary           346355 non-null object\n",
      "unixReviewTime    346355 non-null int64\n",
      "dtypes: int64(2), object(7)\n",
      "memory usage: 26.4+ MB\n"
     ]
    }
   ],
   "source": [
    "# View information about the data\n",
    "raw_df.info()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Raw Corpus contains 178,581,273 characters\n"
     ]
    }
   ],
   "source": [
    "# Convert all the review text into a long string and print its length\n",
    "raw_corpus = u''.join(raw_df['reviewText']+\" \")\n",
    "print(\"Raw Corpus contains {0:,} characters\".format(len(raw_corpus)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[nltk_data] Downloading package punkt to /Users/MacBook/nltk_data...\n",
      "[nltk_data]   Package punkt is already up-to-date!\n",
      "The punkt tokenizer is downloaded\n"
     ]
    }
   ],
   "source": [
    "# import natural language toolkit\n",
    "import nltk\n",
    "# download the punkt tokenizer\n",
    "nltk.download('punkt')\n",
    "print(\"The punkt tokenizer is downloaded\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The punkt tokenizer is loaded\n",
      "We have 1,824,643 raw sentences\n"
     ]
    }
   ],
   "source": [
    "# Load the punkt tokenizer\n",
    "tokenizer = nltk.data.load(\"tokenizers/punkt/english.pickle\")\n",
    "print(\"The punkt tokenizer is loaded\")\n",
    "# we tokenize the raw string into raw sentences\n",
    "raw_sentences = tokenizer.tokenize(raw_corpus)\n",
    "print(\"We have {0:,} raw sentences\".format(len(raw_sentences)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "# Clean and split sentence into words\n",
    "def clean_and_split_str(string):\n",
    "    strip_special_chars = re.compile(\"[^A-Za-z]+\")\n",
    "    string = re.sub(strip_special_chars, \" \", string)\n",
    "    return string.strip().split()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We have 1,824,643 clean sentences\n"
     ]
    }
   ],
   "source": [
    "# clean each raw sentences and build the list of sentences\n",
    "sentences = []\n",
    "for raw_sent in raw_sentences:\n",
    "    if len(raw_sent) > 0:\n",
    "        sentences.append(clean_and_split_str(raw_sent))\n",
    "print(\"We have {0:,} clean sentences\".format(len(sentences)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "I use this magnifier to inspect seeds and leaves.\n",
      "\n",
      "[u'I', u'use', u'this', u'magnifier', u'to', u'inspect', u'seeds', u'and', u'leaves']\n"
     ]
    }
   ],
   "source": [
    "print(raw_sentences[30])\n",
    "print()\n",
    "print(sentences[30])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The dataset corpus contains 33,476,197 tokens\n"
     ]
    }
   ],
   "source": [
    "token_count = sum([len(sentence) for sentence in sentences])\n",
    "print(\"The dataset corpus contains {0:,} tokens\".format(token_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import multiprocessing\n",
    "\n",
    "#Dimensionality of the resulting word vectors\n",
    "num_features = 300\n",
    "\n",
    "#Minimum word count threshold\n",
    "min_word_count = 3\n",
    "\n",
    "#Number of threads to run in parallel\n",
    "num_workers = multiprocessing.cpu_count()\n",
    "\n",
    "#Context window length\n",
    "context_size = 7\n",
    "\n",
    "#Seed for the RNG, to make the result reproducible\n",
    "seed = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gensim\n",
    "\n",
    "word2vec_model = gensim.models.word2vec.Word2Vec(\n",
    "    sg=1,\n",
    "    seed=seed,\n",
    "    workers=num_workers, \n",
    "    size=num_features, \n",
    "    min_count=min_word_count, \n",
    "    window=context_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "The vocabulary is built\n",
      "Word2Vec vocabulary length:  62972\n"
     ]
    }
   ],
   "source": [
    "word2vec_model.build_vocab(sentences=sentences)\n",
    "print(\"The vocabulary is built\")\n",
    "print(\"Word2Vec vocabulary length: \", len(word2vec_model.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training finished\n"
     ]
    }
   ],
   "source": [
    "#Start training the model\n",
    "word2vec_model.train(sentences=sentences)\n",
    "print(\"Training finished\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Model saved\n"
     ]
    }
   ],
   "source": [
    "#Save the model\n",
    "word2vec_model.save(\"word2vec_model_trained_on_Health_and_Personal_Care_5.w2v\")\n",
    "print(\"Model saved\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}