demo.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#future is the missing compatibility layer between Python 2 and Python 3. \n",
    "#It allows you to use a single, clean Python 3.x-compatible codebase to \n",
    "#support both Python 2 and Python 3 with minimal overhead.\n",
    "from __future__ import absolute_import, division, print_function"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#encoding. word encodig\n",
    "import codecs\n",
    "#finds all pathnames matching a pattern, like regex\n",
    "import glob\n",
    "#log events for libraries\n",
    "import logging\n",
    "#concurrency\n",
    "import multiprocessing\n",
    "#dealing with operating system , like reading file\n",
    "import os\n",
    "#pretty print, human readable\n",
    "import pprint\n",
    "#regular expressions\n",
    "import re"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#natural language toolkit\n",
    "import nltk\n",
    "#word 2 vec\n",
    "import gensim.models.word2vec as w2v\n",
    "#dimensionality reduction\n",
    "import sklearn.manifold\n",
    "#math\n",
    "import numpy as np\n",
    "#plotting\n",
    "import matplotlib.pyplot as plt\n",
    "#parse dataset\n",
    "import pandas as pd\n",
    "#visualization\n",
    "import seaborn as sns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "%pylab inline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'nltk' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-1-b1fff18dbd77>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;31m#tokenization into sentences, punkt\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0;31m#http://www.nltk.org/\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 4\u001b[0;31m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"punkt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      5\u001b[0m \u001b[0mnltk\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mdownload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"stopwords\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'nltk' is not defined"
     ]
    }
   ],
   "source": [
    "#stopwords like the at a an, unnecesasry\n",
    "#tokenization into sentences, punkt \n",
    "#http://www.nltk.org/\n",
    "nltk.download(\"punkt\")\n",
    "nltk.download(\"stopwords\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#get the book names, matching txt file\n",
    "book_filenames = sorted(glob.glob(\"/*.txt\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print books\n",
    "print(\"Found books:\")\n",
    "book_filenames"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#step 1 process data\n",
    "\n",
    "#initialize rawunicode , we'll add all text to this one bigass file in memory\n",
    "corpus_raw = u\"\"\n",
    "#for each book, read it, open it un utf 8 format, \n",
    "#add it to the raw corpus\n",
    "for book_filename in book_filenames:\n",
    "    print(\"Reading '{0}'...\".format(book_filename))\n",
    "    with codecs.open(book_filename, \"r\", \"utf-8\") as book_file:\n",
    "        corpus_raw += book_file.read()\n",
    "    print(\"Corpus is now {0} characters long\".format(len(corpus_raw)))\n",
    "    print()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#tokenizastion! saved the trained model here\n",
    "tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#tokenize into sentences\n",
    "raw_sentences = tokenizer.tokenize(corpus_raw)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#convert into list of words\n",
    "#remove unecessary characters, split into words, no hyhens and shit\n",
    "#split into words\n",
    "def sentence_to_wordlist(raw):\n",
    "    clean = re.sub(\"[^a-zA-Z]\",\" \", raw)\n",
    "    words = clean.split()\n",
    "    return words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#for each sentece, sentences where each word is tokenized\n",
    "sentences = []\n",
    "for raw_sentence in raw_sentences:\n",
    "    if len(raw_sentence) > 0:\n",
    "        sentences.append(sentence_to_wordlist(raw_sentence))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#print an example\n",
    "print(raw_sentences[5])\n",
    "print(sentence_to_wordlist(raw_sentences[5]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#count tokens, each one being a sentence\n",
    "token_count = sum([len(sentence) for sentence in sentences])\n",
    "print(\"The book corpus contains {0:,} tokens\".format(token_count))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#step 2 build our model, another one is Glove\n",
    "#define hyperparameters\n",
    "\n",
    "# Dimensionality of the resulting word vectors.\n",
    "#more dimensions mean more traiig them, but more generalized\n",
    "num_features = 300\n",
    "\n",
    "#\n",
    "# Minimum word count threshold.\n",
    "min_word_count = 3\n",
    "\n",
    "# Number of threads to run in parallel.\n",
    "num_workers = multiprocessing.cpu_count()\n",
    "\n",
    "# Context window length.\n",
    "context_size = 7\n",
    "\n",
    "# Downsample setting for frequent words.\n",
    "#rate 0 and 1e-5 \n",
    "#how often to use\n",
    "downsampling = 1e-3\n",
    "\n",
    "# Seed for the RNG, to make the results reproducible.\n",
    "seed = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thrones2vec = w2v.Word2Vec(\n",
    "    sg=1,\n",
    "    seed=seed,\n",
    "    workers=num_workers,\n",
    "    size=num_features,\n",
    "    min_count=min_word_count,\n",
    "    window=context_size,\n",
    "    sample=downsampling\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thrones2vec.build_vocab(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print(\"Word2Vec vocabulary length:\", len(thrones2vec.vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train model on sentneces\n",
    "thrones2vec.train(sentences)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#save model\n",
    "if not os.path.exists(\"trained\"):\n",
    "    os.makedirs(\"trained\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thrones2vec.save(os.path.join(\"trained\", \"thrones2vec.w2v\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#load model\n",
    "thrones2vec = w2v.Word2Vec.load(os.path.join(\"trained\", \"thrones2vec.w2v\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#squash dimensionality to 2\n",
    "#https://www.oreilly.com/learning/an-illustrated-introduction-to-the-t-sne-algorithm\n",
    "tsne = sklearn.manifold.TSNE(n_components=2, random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#put it all into a giant matrix\n",
    "all_word_vectors_matrix = thrones2vec.syn0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#train t sne\n",
    "all_word_vectors_matrix_2d = tsne.fit_transform(all_word_vectors_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#plot point in 2d space\n",
    "points = pd.DataFrame(\n",
    "    [\n",
    "        (word, coords[0], coords[1])\n",
    "        for word, coords in [\n",
    "            (word, all_word_vectors_matrix_2d[thrones2vec.vocab[word].index])\n",
    "            for word in thrones2vec.vocab\n",
    "        ]\n",
    "    ],\n",
    "    columns=[\"word\", \"x\", \"y\"]\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "points.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#plot\n",
    "sns.set_context(\"poster\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "points.plot.scatter(\"x\", \"y\", s=10, figsize=(20, 12))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def plot_region(x_bounds, y_bounds):\n",
    "    slice = points[\n",
    "        (x_bounds[0] <= points.x) &\n",
    "        (points.x <= x_bounds[1]) & \n",
    "        (y_bounds[0] <= points.y) &\n",
    "        (points.y <= y_bounds[1])\n",
    "    ]\n",
    "    \n",
    "    ax = slice.plot.scatter(\"x\", \"y\", s=35, figsize=(10, 8))\n",
    "    for i, point in slice.iterrows():\n",
    "        ax.text(point.x + 0.005, point.y + 0.005, point.word, fontsize=11)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_region(x_bounds=(4.0, 4.2), y_bounds=(-0.5, -0.1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "plot_region(x_bounds=(0, 1), y_bounds=(4, 4.5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thrones2vec.most_similar(\"Stark\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thrones2vec.most_similar(\"Aerys\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "thrones2vec.most_similar(\"direwolf\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "#distance, similarity, and ranking\n",
    "def nearest_similarity_cosmul(start1, end1, end2):\n",
    "    similarities = thrones2vec.most_similar_cosmul(\n",
    "        positive=[end2, start1],\n",
    "        negative=[end1]\n",
    "    )\n",
    "    start2 = similarities[0][0]\n",
    "    print(\"{start1} is related to {end1}, as {start2} is related to {end2}\".format(**locals()))\n",
    "    return start2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "nearest_similarity_cosmul(\"Stark\", \"Winterfell\", \"Riverrun\")\n",
    "nearest_similarity_cosmul(\"Jaime\", \"sword\", \"wine\")\n",
    "nearest_similarity_cosmul(\"Arya\", \"Nymeria\", \"dragons\")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}