Lab3_tfidf.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import string"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u\"======== 01-01\\nWritten by: Marta Kauffman & David Crane\\nMonica: There's nothing to tell! He's just some guy I work with!\\nJoey: C'mon, you're going out with the guy! There's gotta be something wrong wi\""
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "contents = unicode(open('data/friends_transcripts.txt').read(), errors='ignore')\n",
    "contents[:200]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "First element of ``re.split`` is empty string, so skip it:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "666\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "splitted = re.split('======== (\\d\\d)-(\\d\\d)\\n', contents)[1:]\n",
    "print(len(splitted))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from itertools import izip\n",
    "\n",
    "def tripletswise(t):\n",
    "    it = iter(t)\n",
    "    return izip(it,it,it)\n",
    "\n",
    "def clean(txt):\n",
    "    for i in string.punctuation:\n",
    "        txt = txt.replace(i, '')\n",
    "    return txt\n",
    "\n",
    "matched_data = [(int(x), int(y), clean(z)) for x, y, z in tripletswise(splitted)]\n",
    "N = len(matched_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "The first element is **season**,\n",
    "\n",
    "The second is **episode**,\n",
    "\n",
    "And the third – actual **transcript**:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import nltk\n",
    "from nltk.corpus import stopwords\n",
    "from nltk.stem.porter import PorterStemmer\n",
    "\n",
    "def stem_data(data, use_stopwords=True):\n",
    "    def tokenize(doc):        \n",
    "        tokens = nltk.word_tokenize(doc.lower())\n",
    "        if use_stopwords:\n",
    "            return [t for t in tokens if not t in stopwords.words('english')]\n",
    "        else:\n",
    "            return tokens\n",
    "    \n",
    "    def stem(doc):\n",
    "        stemmer = PorterStemmer()\n",
    "        return [stemmer.stem(t) for t in tokenize(doc)]\n",
    "        \n",
    "    return [(seas, ep, stem(doc)) for seas, ep, doc in data]\n",
    "\n",
    "\n",
    "stemmed_data_w_stopwords = stem_data(matched_data, True)\n",
    "stemmed_data_wout_stopwords = stem_data(matched_data, False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[u'written', u'marta', u'kauffman', u'david', u'crane', u'monica', u'there', u'noth', u'tell', u'he']\n",
      "[u'written', u'by', u'marta', u'kauffman', u'david', u'crane', u'monica', u'there', u'noth', u'to']\n"
     ]
    }
   ],
   "source": [
    "print(stemmed_data_w_stopwords[0][2][:10])\n",
    "print(stemmed_data_wout_stopwords[0][2][:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from collections import Counter\n",
    "\n",
    "def count_doc_frequencies(data):\n",
    "    doc_frequencies = Counter()\n",
    "    for _, _, doc in data:\n",
    "        for w in set(doc):\n",
    "            doc_frequencies[w] += 1\n",
    "    \n",
    "    return doc_frequencies\n",
    "\n",
    "\n",
    "docfreq_w_stopwords = count_doc_frequencies(stemmed_data_w_stopwords)\n",
    "docfreq_wout_stopwords = count_doc_frequencies(stemmed_data_wout_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(u'end', 216), (u'ross', 214), (u'well', 212), (u'dont', 212), (u'im', 211), (u'joey', 211), (u'oh', 211), (u'know', 210), (u'hey', 209), (u'your', 209)]\n",
      "[(u'by', 217), (u'end', 216), (u'you', 214), (u'ross', 214), (u'the', 213), (u'and', 213), (u'well', 212), (u'just', 212), (u'dont', 212), (u'are', 212)]\n"
     ]
    }
   ],
   "source": [
    "import operator\n",
    "\n",
    "def sortdict(d):\n",
    "    return sorted(d.items(), key=operator.itemgetter(1), reverse=True)\n",
    "    \n",
    "print(sortdict(docfreq_w_stopwords)[:10])\n",
    "print(sortdict(docfreq_wout_stopwords)[:10])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import math\n",
    "\n",
    "def calc_tfidf(word, freq, doc_freq):\n",
    "    tf = freq\n",
    "    idf = math.log(N / (doc_freq[word] + 1))\n",
    "    return tf * idf\n",
    "\n",
    "def calculate_tfidf(data, document_frequencies, limit=20):\n",
    "    result = []\n",
    "    for seas, ep, script in data:\n",
    "        metricised = {}\n",
    "        for word, count in Counter(script).iteritems():\n",
    "            metric = calc_tfidf(word, count, document_frequencies)\n",
    "            metricised[word] = metric\n",
    "\n",
    "        tfidf_weighted = sortdict(metricised)\n",
    "    \n",
    "        result.append((seas, ep, tfidf_weighted[:limit]))\n",
    "        \n",
    "    return result\n",
    "        \n",
    "tfidf_w_stopwords = calculate_tfidf(stemmed_data_w_stopwords, docfreq_w_stopwords)\n",
    "tfidf_wout_stopwords = calculate_tfidf(stemmed_data_wout_stopwords, docfreq_wout_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Season 1, episode 1: paul: 117.460, franni: 25.824, cut: 19.775, la: 15.390, goodnight: 12.712, spoon: 12.364, stair: 12.364, cmon:  9.730, bracketi:  9.419, bookcas:  9.419, boot:  9.273, wine:  8.959, push:  8.959, vega:  8.671, grandchildren:  8.608, aura:  8.608, shoe:  8.047, joann:  8.015, flavor:  8.015, aruba:  8.015\n",
      "\n",
      "Season 1, episode 2: barri: 56.664, carol: 46.674, susan: 32.236, robbi: 28.257, marsha: 23.548, helen: 22.705, geller: 20.874, ew: 20.520, comedian: 18.838, oberman: 17.216, julia: 14.129, minni: 14.129, mr: 10.397, mindi:  9.940, marlon:  9.419, twirli:  9.419, doy:  9.419, chaotic:  9.419, ludwin:  9.419, fluf:  9.419\n",
      "\n",
      "Season 1, episode 3: alan: 87.036, lizzi: 48.088, paula: 28.257, smoke: 22.874, notnotmin: 18.838, chi: 16.029, thumb: 15.890, farm: 15.137, notmin: 14.129, crook: 14.129, cell:  9.888, basealan:  9.419, schhteve:  9.419, knucklecrack:  9.419, millner:  9.419, lambchop:  9.419, damon:  9.419, coyot:  9.419, poptart:  9.419, thousand:  9.210\n",
      "\n",
      "Season 1, episode 4: joann: 24.044, omnipot: 23.548, kiki: 23.548, receptionist: 21.666, pizza: 19.709, peach: 17.216, hockey: 14.444, jack: 13.816, georg: 13.183, twentieth: 12.912, puck: 12.912, lesli: 12.712, cmon:  9.730, carol:  9.657, bean:  9.534, blobbi:  9.419, leetch:  9.419, bonehead:  9.419, twister:  9.419, messier:  9.419\n",
      "\n",
      "Season 1, episode 5: angela: 64.997, bob: 39.586, janic: 33.271, laundri: 31.794, sud: 23.548, ok: 23.071, la: 17.955, bullwinkl: 14.129, machin: 12.542, mississippi: 12.022, cart: 11.353, boom: 10.302, merced:  9.419, basket:  9.273, uhoh:  8.959, espresso:  8.608, squirrel:  8.608, toni:  8.500, sock:  8.124, horribl:  8.047\n",
      "\n"
     ]
    }
   ],
   "source": [
    "def print_tfidf_data(tfidf_data):\n",
    "    for seas, ep, metrics in tfidf_data:\n",
    "        freq = \", \".join(\"{}: {:6.3f}\".format(x, y) for x, y in metrics)\n",
    "        print(\"Season {}, episode {}: {}\".format(seas, ep, freq))\n",
    "        print\n",
    "        \n",
    "print_tfidf_data(tfidf_w_stopwords[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" cellpadding=\"3\" cellspacing=\"0\"  style=\"border:1px solid black;border-collapse:collapse;\"><tr><td>paul</td><td>117.4596</td></tr><tr><td>franni</td><td>25.8244</td></tr><tr><td>cut</td><td>19.7750</td></tr><tr><td>la</td><td>15.3897</td></tr><tr><td>goodnight</td><td>12.7122</td></tr><tr><td>spoon</td><td>12.3642</td></tr><tr><td>stair</td><td>12.3642</td></tr><tr><td>cmon</td><td>9.7296</td></tr><tr><td>bracketi</td><td>9.4191</td></tr><tr><td>bookcas</td><td>9.4191</td></tr><tr><td>boot</td><td>9.2731</td></tr><tr><td>wine</td><td>8.9588</td></tr><tr><td>push</td><td>8.9588</td></tr><tr><td>vega</td><td>8.6711</td></tr><tr><td>grandchildren</td><td>8.6081</td></tr><tr><td>aura</td><td>8.6081</td></tr><tr><td>shoe</td><td>8.0472</td></tr><tr><td>joann</td><td>8.0147</td></tr><tr><td>flavor</td><td>8.0147</td></tr><tr><td>aruba</td><td>8.0147</td></tr></table>"
      ],
      "text/plain": [
       "<ipy_table.IpyTable at 0x10e1d1f10>"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from ipy_table import make_table\n",
    "\n",
    "make_table(tfidf_w_stopwords[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Season 1, episode 1: paul: 117.460, franni: 25.824, cut: 19.775, la: 15.390, goodnight: 12.712, spoon: 12.364, stair: 12.364, cmon:  9.730, bracketi:  9.419, bookcas:  9.419, boot:  9.273, wine:  8.959, push:  8.959, vega:  8.671, grandchildren:  8.608, aura:  8.608, shoe:  8.047, joann:  8.015, flavor:  8.015, aruba:  8.015\n",
      "\n",
      "Season 1, episode 2: barri: 56.664, carol: 46.674, susan: 32.236, robbi: 28.257, marsha: 23.548, helen: 22.705, geller: 20.874, ew: 20.520, comedian: 18.838, oberman: 17.216, julia: 14.129, minni: 14.129, mr: 10.397, mindi:  9.940, twirli:  9.419, doy:  9.419, marlon:  9.419, chaotic:  9.419, ludwin:  9.419, fluf:  9.419\n",
      "\n",
      "Season 1, episode 3: alan: 87.036, lizzi: 48.088, paula: 28.257, smoke: 22.874, notnotmin: 18.838, chi: 16.029, thumb: 15.890, farm: 15.137, notmin: 14.129, crook: 14.129, cell:  9.888, basealan:  9.419, schhteve:  9.419, knucklecrack:  9.419, millner:  9.419, lambchop:  9.419, damon:  9.419, coyot:  9.419, poptart:  9.419, thousand:  9.210\n",
      "\n",
      "Season 1, episode 4: joann: 24.044, omnipot: 23.548, kiki: 23.548, receptionist: 21.666, pizza: 19.709, peach: 17.216, hockey: 14.444, jack: 13.816, georg: 13.183, twentieth: 12.912, puck: 12.912, lesli: 12.712, cmon:  9.730, carol:  9.657, bean:  9.534, blobbi:  9.419, leetch:  9.419, bonehead:  9.419, twister:  9.419, messier:  9.419\n",
      "\n",
      "Season 1, episode 5: angela: 64.997, bob: 39.586, janic: 33.271, laundri: 31.794, sud: 23.548, ok: 23.071, la: 17.955, bullwinkl: 14.129, machin: 12.542, mississippi: 12.022, cart: 11.353, boom: 10.302, merced:  9.419, basket:  9.273, uhoh:  8.959, espresso:  8.608, squirrel:  8.608, toni:  8.500, sock:  8.124, horribl:  8.047\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print_tfidf_data(tfidf_wout_stopwords[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" cellpadding=\"3\" cellspacing=\"0\"  style=\"border:1px solid black;border-collapse:collapse;\"><tr><td>paul</td><td>117.4596</td></tr><tr><td>franni</td><td>25.8244</td></tr><tr><td>cut</td><td>19.7750</td></tr><tr><td>la</td><td>15.3897</td></tr><tr><td>goodnight</td><td>12.7122</td></tr><tr><td>spoon</td><td>12.3642</td></tr><tr><td>stair</td><td>12.3642</td></tr><tr><td>cmon</td><td>9.7296</td></tr><tr><td>bracketi</td><td>9.4191</td></tr><tr><td>bookcas</td><td>9.4191</td></tr><tr><td>boot</td><td>9.2731</td></tr><tr><td>wine</td><td>8.9588</td></tr><tr><td>push</td><td>8.9588</td></tr><tr><td>vega</td><td>8.6711</td></tr><tr><td>grandchildren</td><td>8.6081</td></tr><tr><td>aura</td><td>8.6081</td></tr><tr><td>shoe</td><td>8.0472</td></tr><tr><td>joann</td><td>8.0147</td></tr><tr><td>flavor</td><td>8.0147</td></tr><tr><td>aruba</td><td>8.0147</td></tr></table>"
      ],
      "text/plain": [
       "<ipy_table.IpyTable at 0x103c4b650>"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_table(tfidf_wout_stopwords[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(1, 2, [(u'barri', 56.66426688112432), (u'carol', 46.67369946058891), (u'susan', 32.23619130191664), (u'robbi', 28.257181207874005), (u'marsha', 23.54765100656167), (u'helen', 22.705137803509565), (u'geller', 20.873633484694086), (u'ew', 20.519594859692294), (u'comedian', 18.838120805249336), (u'oberman', 17.21626037281668), (u'julia', 14.128590603937003), (u'minni', 14.128590603937003), (u'mr', 10.39720770839918), (u'mindi', 9.939626599152001), (u'marlon', 9.419060402624668), (u'twirli', 9.419060402624668), (u'doy', 9.419060402624668), (u'chaotic', 9.419060402624668), (u'ludwin', 9.419060402624668), (u'fluf', 9.419060402624668)])\n",
      "(1, 2, [(u'barri', 56.66426688112432), (u'carol', 46.67369946058891), (u'susan', 32.23619130191664), (u'robbi', 28.257181207874005), (u'marsha', 23.54765100656167), (u'helen', 22.705137803509565), (u'geller', 20.873633484694086), (u'ew', 20.519594859692294), (u'comedian', 18.838120805249336), (u'oberman', 17.21626037281668), (u'julia', 14.128590603937003), (u'minni', 14.128590603937003), (u'mr', 10.39720770839918), (u'mindi', 9.939626599152001), (u'twirli', 9.419060402624668), (u'doy', 9.419060402624668), (u'marlon', 9.419060402624668), (u'chaotic', 9.419060402624668), (u'ludwin', 9.419060402624668), (u'fluf', 9.419060402624668)])\n",
      "\n",
      "(1, 12, [(u'paolo', 61.81176968073263), (u'carol', 24.141568686511505), (u'susan', 23.02585092994046), (u'ok', 19.775021196025975), (u'pocono', 18.838120805249336), (u'lasagna', 18.05458956322112), (u'kip', 12.912195279612511), (u'pig', 12.364169813433264), (u'bird', 9.729550745276565), (u'syl', 9.419060402624668), (u'mma', 9.419060402624668), (u'racquela', 9.419060402624668), (u'feminist', 9.419060402624668), (u'vegetarian', 9.273127360074948), (u'ho', 8.987196820661973), (u'immigr', 8.60813018640834), (u'fling', 8.014666370464942), (u'nauseou', 8.014666370464942), (u'boy', 7.6246189861593985), (u'swoop', 6.3561076606958915)])\n",
      "(1, 12, [(u'paolo', 61.81176968073263), (u'carol', 24.141568686511505), (u'susan', 23.02585092994046), (u'ok', 19.775021196025975), (u'pocono', 18.838120805249336), (u'lasagna', 18.05458956322112), (u'kip', 12.912195279612511), (u'pig', 12.364169813433264), (u'bird', 9.729550745276565), (u'mma', 9.419060402624668), (u'racquela', 9.419060402624668), (u'syl', 9.419060402624668), (u'feminist', 9.419060402624668), (u'vegetarian', 9.273127360074948), (u'ho', 8.987196820661973), (u'immigr', 8.60813018640834), (u'fling', 8.014666370464942), (u'nauseou', 8.014666370464942), (u'boy', 7.6246189861593985), (u'swoop', 6.3561076606958915)])\n",
      "\n",
      "(1, 13, [(u'ronni', 122.44778523412069), (u'roger', 104.7166194666825), (u'tribbiani', 36.25420552604762), (u'boobi', 28.051332296627297), (u'mr', 22.873856958478193), (u'weebl', 18.838120805249336), (u'rog', 14.128590603937003), (u'failur', 14.128590603937003), (u'sibl', 14.128590603937003), (u'cmon', 13.621371043387192), (u'dad', 10.986122886681098), (u'whatr', 10.259797429846147), (u'tat', 9.419060402624668), (u'nip', 9.419060402624668), (u'peepe', 8.60813018640834), (u'trib', 8.60813018640834), (u'ship', 8.49964003216865), (u'okaaay', 8.014666370464942), (u'ysee', 7.568379267836522), (u'whoah', 6.8679744089702925)])\n",
      "(1, 13, [(u'ronni', 122.44778523412069), (u'roger', 104.7166194666825), (u'tribbiani', 36.25420552604762), (u'ma', 34.43252074563336), (u'boobi', 28.051332296627297), (u'mr', 22.873856958478193), (u'weebl', 18.838120805249336), (u'rog', 14.128590603937003), (u'failur', 14.128590603937003), (u'sibl', 14.128590603937003), (u'cmon', 13.621371043387192), (u'dad', 10.986122886681098), (u'whatr', 10.259797429846147), (u'tat', 9.419060402624668), (u'nip', 9.419060402624668), (u'peepe', 8.60813018640834), (u'trib', 8.60813018640834), (u'ship', 8.49964003216865), (u'okaaay', 8.014666370464942), (u'ysee', 7.568379267836522)])\n",
      "\n",
      "(1, 15, [(u'steve', 77.88903911240376), (u'celia', 47.09530201312334), (u'tartlet', 23.54765100656167), (u'ok', 20.873633484694086), (u'ahem', 17.21626037281668), (u'cuddl', 12.021999555697413), (u'wenu', 12.021999555697413), (u'huhhuh', 12.021999555697413), (u'waitress', 10.986122886681098), (u'alright', 10.986122886681098), (u'dirti', 10.39720770839918), (u'yummi', 9.534161491043838), (u'er', 9.534161491043838), (u'agogo', 9.419060402624668), (u'vulva', 9.419060402624668), (u'tedlock', 9.419060402624668), (u'jeff', 8.60813018640834), (u'sooth', 8.014666370464942), (u'counselor', 8.014666370464942), (u'mouss', 8.014666370464942)])\n",
      "(1, 15, [(u'steve', 77.88903911240376), (u'celia', 47.09530201312334), (u'tartlet', 23.54765100656167), (u'ok', 20.873633484694086), (u'ahem', 17.21626037281668), (u'cuddl', 12.021999555697413), (u'wenu', 12.021999555697413), (u'huhhuh', 12.021999555697413), (u'waitress', 10.986122886681098), (u'alright', 10.986122886681098), (u'dirti', 10.39720770839918), (u'yummi', 9.534161491043838), (u'er', 9.534161491043838), (u'agogo', 9.419060402624668), (u'vulva', 9.419060402624668), (u'tedlock', 9.419060402624668), (u'jeff', 8.60813018640834), (u'mouss', 8.014666370464942), (u'sooth', 8.014666370464942), (u'counselor', 8.014666370464942)])\n",
      "\n",
      "(1, 18, [(u'poker', 64.11733096371954), (u'iri', 32.96671140918634), (u'ok', 31.859756371375184), (u'alright', 26.366694928034633), (u'mee', 14.128590603937003), (u'card', 12.875503299472802), (u'aunt', 12.542316284596385), (u'bluf', 12.021999555697413), (u'ooo', 11.352568901754783), (u'rais', 10.75055681536833), (u'ha', 10.75055681536833), (u'dammit', 10.30196161345544), (u'fold', 9.887510598012987), (u'dealt', 9.419060402624668), (u'pictionari', 9.419060402624668), (u'750', 9.419060402624668), (u'yaya', 9.419060402624668), (u'fifteen', 9.273127360074948), (u'cent', 9.273127360074948), (u'airport', 9.210340371976184)])\n",
      "(1, 18, [(u'poker', 64.11733096371954), (u'iri', 32.96671140918634), (u'ok', 31.859756371375184), (u'alright', 26.366694928034633), (u'mee', 14.128590603937003), (u'card', 12.875503299472802), (u'aunt', 12.542316284596385), (u'bluf', 12.021999555697413), (u'ooo', 11.352568901754783), (u'rais', 10.75055681536833), (u'dammit', 10.30196161345544), (u'fold', 9.887510598012987), (u'dealt', 9.419060402624668), (u'pictionari', 9.419060402624668), (u'750', 9.419060402624668), (u'yaya', 9.419060402624668), (u'fifteen', 9.273127360074948), (u'cent', 9.273127360074948), (u'airport', 9.210340371976184), (u'lesson', 8.49964003216865)])\n",
      "\n",
      "(2, 1, [(u'juli', 56.428885864153806), (u'moor', 28.257181207874005), (u'paulo', 28.051332296627297), (u'franki', 23.54765100656167), (u'ok', 21.972245773362197), (u'mcdowel', 18.838120805249336), (u'demi', 18.838120805249336), (u'andi', 12.912195279612511), (u'china', 11.982929094215963), (u'shorter', 11.352568901754783), (u'tailor', 9.419060402624668), (u'airport', 9.210340371976184), (u'lemonad', 8.60813018640834), (u'alter', 8.60813018640834), (u'inseam', 8.60813018640834), (u'cramp', 8.014666370464942), (u'bleed', 7.221835825288449), (u'silver', 7.221835825288449), (u'plane', 6.591673732008658), (u'hair', 6.591673732008658)])\n",
      "(2, 1, [(u'juli', 56.428885864153806), (u'moor', 28.257181207874005), (u'paulo', 28.051332296627297), (u'franki', 23.54765100656167), (u'ok', 21.972245773362197), (u'mcdowel', 18.838120805249336), (u'demi', 18.838120805249336), (u'andi', 12.912195279612511), (u'china', 11.982929094215963), (u'shorter', 11.352568901754783), (u'tailor', 9.419060402624668), (u'airport', 9.210340371976184), (u'lemonad', 8.60813018640834), (u'alter', 8.60813018640834), (u'inseam', 8.60813018640834), (u'cramp', 8.014666370464942), (u'silver', 7.221835825288449), (u'bleed', 7.221835825288449), (u'plane', 6.591673732008658), (u'hair', 6.591673732008658)])\n",
      "\n",
      "(2, 4, [(u'jung', 3.1780538303479458), (u'alexa', 3.1780538303479458), (u'hodg', 2.995732273553991), (u'josh', 2.8903717578961645), (u'twice', 1.9459101490553132), (u'origin', 1.791759469228055), (u'morn', 0.6931471805599453), (u'somebodi', 0.6931471805599453), (u'good', 0.0), (u'end', 0.0), (u'got', 0.0), (u'well', 0.0), (u'transcrib', 0.0), (u'written', 0.0), (u'woman', 0.0), (u'night', 0.0), (u'last', 0.0), (u'old', 0.0), (u'ross', 0.0)])\n",
      "(2, 4, [(u'jung', 3.1780538303479458), (u'alexa', 3.1780538303479458), (u'hodg', 2.995732273553991), (u'josh', 2.8903717578961645), (u'twice', 1.9459101490553132), (u'origin', 1.791759469228055), (u'morn', 0.6931471805599453), (u'somebodi', 0.6931471805599453), (u'good', 0.0), (u'last', 0.0), (u'got', 0.0), (u'well', 0.0), (u'end', 0.0), (u'some', 0.0), (u'transcrib', 0.0), (u'written', 0.0), (u'woman', 0.0), (u'night', 0.0), (u'ross', 0.0), (u'old', 0.0)])\n",
      "\n",
      "(2, 7, [(u'closur', 4.30406509320417), (u'borkow', 3.091042453358316), (u'hodg', 2.995732273553991), (u'josh', 2.8903717578961645), (u'phone', 2.772588722239781), (u'toy', 2.70805020110221), (u'juli', 2.5649493574615367), (u'michael', 2.302585092994046), (u'downstair', 2.1972245773362196), (u'cat', 1.9459101490553132), (u'cab', 1.9459101490553132), (u'origin', 1.791759469228055), (u'dream', 1.791759469228055), (u'ok', 1.0986122886681098), (u'hang', 0.6931471805599453), (u'ive', 0.0), (u'dont', 0.0), (u'give', 0.0), (u'feel', 0.0), (u'say', 0.0)])\n",
      "(2, 7, [(u'closur', 4.30406509320417), (u'borkow', 3.091042453358316), (u'hodg', 2.995732273553991), (u'josh', 2.8903717578961645), (u'phone', 2.772588722239781), (u'toy', 2.70805020110221), (u'juli', 2.5649493574615367), (u'michael', 2.302585092994046), (u'downstair', 2.1972245773362196), (u'cab', 1.9459101490553132), (u'cat', 1.9459101490553132), (u'origin', 1.791759469228055), (u'dream', 1.791759469228055), (u'ok', 1.0986122886681098), (u'hang', 0.6931471805599453), (u'and', 0.0), (u'dont', 0.0), (u'just', 0.0), (u'give', 0.0), (u'feel', 0.0)])\n",
      "\n",
      "(2, 8, [(u'mnca', 167.85853863496263), (u'rtst', 84.77154362362201), (u'phoe', 81.77723677087923), (u'chan', 65.9061100181878), (u'mockol', 61.22389261706034), (u'juli', 53.86393650669227), (u'ok', 29.662531794038966), (u'diari', 16.029332740929885), (u'recip', 12.021999555697413), (u'anyhoo', 12.021999555697413), (u'chocol', 10.556229318461034), (u'coat', 10.39720770839918), (u'rachem', 9.419060402624668), (u'ditzi', 9.419060402624668), (u'con', 9.419060402624668), (u'megabyt', 9.419060402624668), (u'salin', 9.419060402624668), (u'column', 9.419060402624668), (u'fda', 9.419060402624668), (u'ankl', 8.987196820661973)])\n",
      "(2, 8, [(u'mnca', 167.85853863496263), (u'rtst', 84.77154362362201), (u'phoe', 81.77723677087923), (u'chan', 65.9061100181878), (u'mockol', 61.22389261706034), (u'juli', 53.86393650669227), (u'ok', 29.662531794038966), (u'diari', 16.029332740929885), (u'recip', 12.021999555697413), (u'anyhoo', 12.021999555697413), (u'chocol', 10.556229318461034), (u'coat', 10.39720770839918), (u'rachem', 9.419060402624668), (u'con', 9.419060402624668), (u'megabyt', 9.419060402624668), (u'salin', 9.419060402624668), (u'ditzi', 9.419060402624668), (u'column', 9.419060402624668), (u'fda', 9.419060402624668), (u'ankl', 8.987196820661973)])\n",
      "\n",
      "(2, 10, [(u'mnca', 206.59512447380018), (u'phoe', 163.55447354175845), (u'russ', 160.12402684461935), (u'fbob', 127.15731543543302), (u'chan', 107.84636184794367), (u'bobbi', 31.780538303479457), (u'ok', 29.662531794038966), (u'estl', 28.257181207874005), (u'sober', 14.128590603937003), (u'ewww', 12.364169813433264), (u'alright', 10.986122886681098), (u'gang', 10.30196161345544), (u'dull', 9.887510598012987), (u'uhhh', 9.534161491043838), (u'dysprosium', 9.419060402624668), (u'adolesc', 9.419060402624668), (u'saget', 9.419060402624668), (u'mindless', 9.419060402624668), (u'soooo', 8.014666370464942), (u'hardwar', 8.014666370464942)])\n",
      "(2, 10, [(u'mnca', 206.59512447380018), (u'phoe', 163.55447354175845), (u'russ', 160.12402684461935), (u'fbob', 127.15731543543302), (u'chan', 107.84636184794367), (u'bobbi', 31.780538303479457), (u'ok', 29.662531794038966), (u'estl', 28.257181207874005), (u'sober', 14.128590603937003), (u'ewww', 12.364169813433264), (u'alright', 10.986122886681098), (u'gang', 10.30196161345544), (u'dull', 9.887510598012987), (u'uhhh', 9.534161491043838), (u'dysprosium', 9.419060402624668), (u'mindless', 9.419060402624668), (u'adolesc', 9.419060402624668), (u'saget', 9.419060402624668), (u'soooo', 8.014666370464942), (u'hardwar', 8.014666370464942)])\n",
      "\n",
      "(2, 11, [(u'carol', 38.626509898418405), (u'green', 34.657359027997266), (u'susan', 29.933606208922598), (u'mr', 26.33959286127792), (u'ok', 19.775021196025975), (u'barri', 14.16606672028108), (u'dice', 12.912195279612511), (u'adelman', 9.419060402624668), (u'bitterest', 9.419060402624668), (u'wallac', 9.419060402624668), (u'soupi', 9.419060402624668), (u'unfinish', 9.419060402624668), (u'dermer', 9.419060402624668), (u'evelyn', 8.60813018640834), (u'revel', 8.60813018640834), (u'lamb', 8.014666370464942), (u'melon', 7.568379267836522), (u'option', 6.591673732008658), (u'rose', 6.591673732008658), (u'wed', 6.238324625039508)])\n",
      "(2, 11, [(u'carol', 38.626509898418405), (u'green', 34.657359027997266), (u'susan', 29.933606208922598), (u'mr', 26.33959286127792), (u'ok', 19.775021196025975), (u'barri', 14.16606672028108), (u'dice', 12.912195279612511), (u'unfinish', 9.419060402624668), (u'adelman', 9.419060402624668), (u'bitterest', 9.419060402624668), (u'wallac', 9.419060402624668), (u'soupi', 9.419060402624668), (u'dermer', 9.419060402624668), (u'evelyn', 8.60813018640834), (u'revel', 8.60813018640834), (u'lamb', 8.014666370464942), (u'melon', 7.568379267836522), (u'option', 6.591673732008658), (u'rose', 6.591673732008658), (u'wed', 6.238324625039508)])\n",
      "\n",
      "(2, 14, [(u'trascrib', 4.007333185232471), (u'alexa', 3.1780538303479458), (u'jung', 3.1780538303479458), (u'hodg', 2.995732273553991), (u'josh', 2.8903717578961645), (u'origin', 1.791759469228055), (u'written', 0.0), (u'end', 0.0)])\n",
      "(2, 14, [(u'trascrib', 4.007333185232471), (u'alexa', 3.1780538303479458), (u'jung', 3.1780538303479458), (u'hodg', 2.995732273553991), (u'josh', 2.8903717578961645), (u'origin', 1.791759469228055), (u'written', 0.0), (u'end', 0.0), (u'by', 0.0)])\n",
      "\n",
      "(2, 15, [(u'ok', 5.493061443340549), (u'nuh', 4.709530201312334), (u'ursa', 4.709530201312334), (u'cluster', 4.007333185232471), (u'consciou', 3.784189633918261), (u'self', 3.784189633918261), (u'woah', 3.6109179126442243), (u'hodg', 2.995732273553991), (u'squeez', 2.995732273553991), (u'cheek', 2.8903717578961645), (u'joshua', 2.833213344056216), (u'rub', 2.5649493574615367), (u'major', 2.3978952727983707), (u'warm', 2.302585092994046), (u'near', 2.302585092994046), (u'cmon', 1.9459101490553132), (u'origin', 1.791759469228055), (u'butt', 1.791759469228055), (u'romant', 1.791759469228055), (u'worth', 1.6094379124341003)])\n",
      "(2, 15, [(u'ok', 5.493061443340549), (u'nuh', 4.709530201312334), (u'ursa', 4.709530201312334), (u'cluster', 4.007333185232471), (u'consciou', 3.784189633918261), (u'self', 3.784189633918261), (u'woah', 3.6109179126442243), (u'squeez', 2.995732273553991), (u'hodg', 2.995732273553991), (u'cheek', 2.8903717578961645), (u'joshua', 2.833213344056216), (u'rub', 2.5649493574615367), (u'major', 2.3978952727983707), (u'warm', 2.302585092994046), (u'near', 2.302585092994046), (u'cmon', 1.9459101490553132), (u'origin', 1.791759469228055), (u'romant', 1.791759469228055), (u'butt', 1.791759469228055), (u'worth', 1.6094379124341003)])\n",
      "\n",
      "(2, 16, [(u'betsi', 4.007333185232471), (u'silverstein', 3.1780538303479458), (u'addit', 3.1780538303479458), (u'hodg', 2.995732273553991), (u'joshua', 2.833213344056216), (u'minor', 2.833213344056216), (u'dan', 2.6390573296152584), (u'adjust', 2.6390573296152584), (u'born', 2.1972245773362196), (u'origin', 1.791759469228055), (u'ok', 1.0986122886681098), (u'hous', 1.0986122886681098), (u'coffe', 0.6931471805599453), (u'um', 0.6931471805599453), (u'right', 0.0), (u'dont', 0.0), (u'yeah', 0.0), (u'tonight', 0.0), (u'see', 0.0), (u'im', 0.0)])\n",
      "(2, 16, [(u'betsi', 4.007333185232471), (u'silverstein', 3.1780538303479458), (u'addit', 3.1780538303479458), (u'hodg', 2.995732273553991), (u'joshua', 2.833213344056216), (u'minor', 2.833213344056216), (u'dan', 2.6390573296152584), (u'adjust', 2.6390573296152584), (u'born', 2.1972245773362196), (u'origin', 1.791759469228055), (u'ok', 1.0986122886681098), (u'hous', 1.0986122886681098), (u'coffe', 0.6931471805599453), (u'um', 0.6931471805599453), (u'and', 0.0), (u'right', 0.0), (u'dont', 0.0), (u'yeah', 0.0), (u'tonight', 0.0), (u'see', 0.0)])\n",
      "\n",
      "(2, 17, [(u'hodg', 2.995732273553991), (u'joshua', 2.833213344056216), (u'adam', 2.5649493574615367), (u'nose', 2.1972245773362196), (u'chase', 2.0794415416798357), (u'origin', 1.791759469228055), (u'ok', 1.0986122886681098), (u'line', 1.0986122886681098), (u'end', 0.0), (u'sorri', 0.0), (u'transcrib', 0.0), (u'your', 0.0), (u'uh', 0.0), (u'written', 0.0), (u'phoeb', 0.0), (u'guy', 0.0), (u'ross', 0.0), (u'friend', 0.0)])\n",
      "(2, 17, [(u'hodg', 2.995732273553991), (u'joshua', 2.833213344056216), (u'adam', 2.5649493574615367), (u'nose', 2.1972245773362196), (u'chase', 2.0794415416798357), (u'origin', 1.791759469228055), (u'line', 1.0986122886681098), (u'ok', 1.0986122886681098), (u'onc', 0.6931471805599453), (u'are', 0.0), (u'have', 0.0), (u'your', 0.0), (u'onli', 0.0), (u'end', 0.0), (u'no', 0.0), (u'sorri', 0.0), (u'transcrib', 0.0), (u'to', 0.0), (u'written', 0.0), (u'phoeb', 0.0)])\n",
      "\n",
      "(2, 18, [(u'richard', 6.238324625039507), (u'brisk', 4.30406509320417), (u'origin', 1.791759469228055), (u'speak', 1.6094379124341003), (u'ohh', 0.0), (u'ross', 0.0), (u'oh', 0.0), (u'never', 0.0), (u'hey', 0.0), (u'tonight', 0.0), (u'written', 0.0), (u'let', 0.0), (u'got', 0.0), (u'end', 0.0), (u'man', 0.0)])\n",
      "(2, 18, [(u'richard', 6.238324625039507), (u'brisk', 4.30406509320417), (u'origin', 1.791759469228055), (u'speak', 1.6094379124341003), (u'ohh', 0.0), (u'ross', 0.0), (u'oh', 0.0), (u'of', 0.0), (u'never', 0.0), (u'hey', 0.0), (u'it', 0.0), (u'tonight', 0.0), (u'written', 0.0), (u'let', 0.0), (u'thi', 0.0), (u'got', 0.0), (u'end', 0.0), (u'you', 0.0), (u'man', 0.0), (u'by', 0.0)])\n",
      "\n",
      "(2, 19, [(u'housetrain', 4.709530201312334), (u'ceram', 4.30406509320417), (u'tacki', 4.007333185232471), (u'tile', 3.295836866004329), (u'zoo', 3.1780538303479458), (u'goin', 2.1972245773362196), (u'origin', 1.791759469228055), (u'fake', 1.791759469228055), (u'paid', 1.791759469228055), (u'dog', 1.3862943611198906), (u'buy', 1.0986122886681098), (u'bathroom', 1.0986122886681098), (u'money', 0.6931471805599453), (u'bring', 0.6931471805599453), (u'hate', 0.6931471805599453), (u'understand', 0.6931471805599453), (u'without', 0.6931471805599453), (u'dont', 0.0), (u'one', 0.0), (u'im', 0.0)])\n",
      "(2, 19, [(u'housetrain', 4.709530201312334), (u'ceram', 4.30406509320417), (u'tacki', 4.007333185232471), (u'tile', 3.295836866004329), (u'zoo', 3.1780538303479458), (u'goin', 2.1972245773362196), (u'origin', 1.791759469228055), (u'fake', 1.791759469228055), (u'paid', 1.791759469228055), (u'dog', 1.3862943611198906), (u'buy', 1.0986122886681098), (u'bathroom', 1.0986122886681098), (u'money', 0.6931471805599453), (u'bring', 0.6931471805599453), (u'hate', 0.6931471805599453), (u'understand', 0.6931471805599453), (u'without', 0.6931471805599453), (u'all', 0.0), (u'hold', 0.0), (u'dont', 0.0)])\n",
      "\n",
      "(2, 20, [(u'bert', 21.52032546602085), (u'erni', 3.784189633918261), (u'origin', 1.791759469228055), (u'moment', 1.0986122886681098), (u'ago', 0.6931471805599453), (u'lost', 0.6931471805599453), (u'glad', 0.6931471805599453), (u'old', 0.0), (u'oh', 0.0), (u'hey', 0.0), (u'written', 0.0), (u'im', 0.0), (u'friend', 0.0), (u'phoeb', 0.0), (u'end', 0.0), (u'happen', 0.0), (u'your', 0.0)])\n",
      "(2, 20, [(u'bert', 21.52032546602085), (u'erni', 3.784189633918261), (u'origin', 1.791759469228055), (u'moment', 1.0986122886681098), (u'ago', 0.6931471805599453), (u'glad', 0.6931471805599453), (u'lost', 0.6931471805599453), (u'old', 0.0), (u'just', 0.0), (u'is', 0.0), (u'im', 0.0), (u'happen', 0.0), (u'your', 0.0), (u'what', 0.0), (u'end', 0.0), (u'wa', 0.0), (u'no', 0.0), (u'hey', 0.0), (u'to', 0.0), (u'written', 0.0)])\n",
      "\n",
      "(2, 21, [(u'buckner', 2.5649493574615367), (u'sebastian', 2.5649493574615367), (u'jone', 2.4849066497880004), (u'brian', 2.0794415416798357), (u'origin', 1.791759469228055), (u'excel', 1.791759469228055), (u'end', 0.0), (u'written', 0.0), (u'joey', 0.0)])\n",
      "(2, 21, [(u'buckner', 2.5649493574615367), (u'sebastian', 2.5649493574615367), (u'jone', 2.4849066497880004), (u'brian', 2.0794415416798357), (u'origin', 1.791759469228055), (u'excel', 1.791759469228055), (u'and', 0.0), (u'end', 0.0), (u'written', 0.0), (u'by', 0.0), (u'joey', 0.0)])\n",
      "\n",
      "(2, 22, [(u'alexa', 3.1780538303479458), (u'jung', 3.1780538303479458), (u'origin', 1.791759469228055), (u'almost', 1.0986122886681098), (u'true', 0.6931471805599453), (u'come', 0.0), (u'wow', 0.0), (u'never', 0.0), (u'thing', 0.0), (u'written', 0.0), (u'end', 0.0)])\n",
      "(2, 22, [(u'alexa', 3.1780538303479458), (u'jung', 3.1780538303479458), (u'origin', 1.791759469228055), (u'almost', 1.0986122886681098), (u'true', 0.6931471805599453), (u'come', 0.0), (u'wow', 0.0), (u'never', 0.0), (u'thing', 0.0), (u'written', 0.0), (u'end', 0.0), (u'by', 0.0), (u'those', 0.0)])\n",
      "\n",
      "(2, 23, [(u'mandel', 4.30406509320417), (u'brown', 2.4849066497880004), (u'origin', 1.791759469228055), (u'written', 0.0), (u'end', 0.0)])\n",
      "(2, 23, [(u'mandel', 4.30406509320417), (u'brown', 2.4849066497880004), (u'origin', 1.791759469228055), (u'end', 0.0), (u'written', 0.0), (u'by', 0.0)])\n",
      "\n",
      "(2, 24, [(u'barri', 8.49964003216865), (u'mindi', 7.4547199493640015), (u'ofinsan', 4.709530201312334), (u'exfiance', 4.709530201312334), (u'min', 4.709530201312334), (u'htmled', 4.709530201312334), (u'mandel', 4.30406509320417), (u'nauseou', 4.007333185232471), (u'liver', 4.007333185232471), (u'ungerleid', 3.784189633918261), (u'syphili', 3.784189633918261), (u'ira', 3.6109179126442243), (u'dammit', 3.4339872044851463), (u'chop', 3.4339872044851463), (u'aisl', 3.091042453358316), (u'b', 2.833213344056216), (u'guilti', 2.6390573296152584), (u'guineapig', 2.6390573296152584), (u'200', 2.6390573296152584), (u'brown', 2.4849066497880004)])\n",
      "(2, 24, [(u'barri', 8.49964003216865), (u'mindi', 7.4547199493640015), (u'ofinsan', 4.709530201312334), (u'exfiance', 4.709530201312334), (u'min', 4.709530201312334), (u'htmled', 4.709530201312334), (u'mandel', 4.30406509320417), (u'nauseou', 4.007333185232471), (u'liver', 4.007333185232471), (u'ungerleid', 3.784189633918261), (u'syphili', 3.784189633918261), (u'ira', 3.6109179126442243), (u'chop', 3.4339872044851463), (u'dammit', 3.4339872044851463), (u'aisl', 3.091042453358316), (u'b', 2.833213344056216), (u'guilti', 2.6390573296152584), (u'guineapig', 2.6390573296152584), (u'200', 2.6390573296152584), (u'brown', 2.4849066497880004)])\n",
      "\n",
      "(3, 2, [(u'breezi', 43.0406509320417), (u'whitfield', 32.96671140918634), (u'michel', 29.662531794038962), (u'messag', 22.532130774077405), (u'hummu', 21.52032546602085), (u'outgo', 18.838120805249336), (u'machin', 16.125835223052494), (u'underwear', 15.567281192442506), (u'tick', 15.136758535673044), (u'chair', 14.484941211906902), (u'dress', 14.281959752685427), (u'opposit', 13.735948817940585), (u'commando', 12.912195279612511), (u'fat', 12.476649250079015), (u'essenc', 12.021999555697413), (u'cushion', 11.352568901754783), (u'um', 11.090354888959125), (u'black', 10.986122886681098), (u'richard', 10.39720770839918), (u'wa', 10.30196161345544)])\n",
      "(3, 2, [(u'breezi', 43.0406509320417), (u'whitfield', 32.96671140918634), (u'michel', 29.662531794038962), (u'messag', 22.532130774077405), (u'hummu', 21.52032546602085), (u'outgo', 18.838120805249336), (u'machin', 16.125835223052494), (u'underwear', 15.567281192442506), (u'tick', 15.136758535673044), (u'chair', 14.484941211906902), (u'dress', 14.281959752685427), (u'opposit', 13.735948817940585), (u'commando', 12.912195279612511), (u'fat', 12.476649250079015), (u'essenc', 12.021999555697413), (u'cushion', 11.352568901754783), (u'um', 11.090354888959125), (u'black', 10.986122886681098), (u'richard', 10.39720770839918), (u'wear', 9.704060527839234)])\n",
      "\n",
      "(3, 16, [(u'issac', 77.47317167767507), (u'chloe', 68.12466414895201), (u'jasmin', 44.08066503755718), (u'wax', 25.27642538850957), (u'trail', 23.54765100656167), (u'mark', 19.45910149055313), (u'waxin', 14.128590603937003), (u'ceil', 10.832753737932673), (u'pizza', 10.75055681536833), (u'girlfriend', 9.704060527839234), (u'ow', 9.656627474604601), (u'owowowow', 9.419060402624668), (u'strip', 9.273127360074948), (u'linen', 8.60813018640834), (u'jeopardis', 8.60813018640834), (u'rebuild', 8.60813018640834), (u'anchovi', 8.60813018640834), (u'pain', 7.783640596221253), (u'bail', 7.568379267836522), (u'mop', 7.568379267836522)])\n",
      "(3, 16, [(u'issac', 77.47317167767507), (u'chloe', 68.12466414895201), (u'jasmin', 44.08066503755718), (u'wax', 25.27642538850957), (u'trail', 23.54765100656167), (u'mark', 19.45910149055313), (u'waxin', 14.128590603937003), (u'ceil', 10.832753737932673), (u'pizza', 10.75055681536833), (u'girlfriend', 9.704060527839234), (u'ow', 9.656627474604601), (u'owowowow', 9.419060402624668), (u'strip', 9.273127360074948), (u'jeopardis', 8.60813018640834), (u'linen', 8.60813018640834), (u'rebuild', 8.60813018640834), (u'anchovi', 8.60813018640834), (u'pain', 7.783640596221253), (u'bail', 7.568379267836522), (u'mop', 7.568379267836522)])\n",
      "\n",
      "(3, 23, [(u'vinc', 122.44778523412069), (u'jason', 90.82055121403826), (u'pete', 76.2732919283507), (u'guru', 70.64295301968501), (u'saj', 65.93342281837268), (u'rhode', 28.257181207874005), (u'sensit', 18.546254720149896), (u'fireman', 17.16993602242573), (u'nippl', 16.24830120661326), (u'conquer', 14.128590603937003), (u'koondi', 14.128590603937003), (u'iveiv', 11.352568901754783), (u'third', 10.75055681536833), (u'plant', 10.30196161345544), (u'charcoal', 9.419060402624668), (u'burli', 9.419060402624668), (u'pimpl', 9.419060402624668), (u'abnorm', 9.419060402624668), (u'news', 8.788898309344878), (u'field', 8.671115273688493)])\n",
      "(3, 23, [(u'vinc', 122.44778523412069), (u'jason', 90.82055121403826), (u'pete', 76.2732919283507), (u'guru', 70.64295301968501), (u'saj', 65.93342281837268), (u'rhode', 28.257181207874005), (u'sensit', 18.546254720149896), (u'fireman', 17.16993602242573), (u'nippl', 16.24830120661326), (u'conquer', 14.128590603937003), (u'koondi', 14.128590603937003), (u'iveiv', 11.352568901754783), (u'third', 10.75055681536833), (u'plant', 10.30196161345544), (u'charcoal', 9.419060402624668), (u'pimpl', 9.419060402624668), (u'burli', 9.419060402624668), (u'abnorm', 9.419060402624668), (u'news', 8.788898309344878), (u'field', 8.671115273688493)])\n",
      "\n",
      "(4, 3, [(u'joanna', 101.10570155403828), (u'salesman', 54.163768689663364), (u'sophi', 19.775021196025975), (u'geller', 19.775021196025975), (u'quich', 18.838120805249336), (u'nail', 17.974393641323946), (u'freedom', 17.21626037281668), (u'hire', 16.785266909588593), (u'walkman', 14.128590603937003), (u'volcano', 12.912195279612511), (u'mr', 12.476649250079015), (u'lasagna', 10.832753737932673), (u'encyclopedia', 9.419060402624668), (u'milton', 9.419060402624668), (u'berl', 9.419060402624668), (u'unlock', 8.60813018640834), (u'pull', 8.317766166719343), (u'key', 8.047189562170502), (u'lock', 7.783640596221253), (u'offic', 7.690286020676768)])\n",
      "(4, 3, [(u'joanna', 101.10570155403828), (u'salesman', 54.163768689663364), (u'sophi', 19.775021196025975), (u'geller', 19.775021196025975), (u'quich', 18.838120805249336), (u'nail', 17.974393641323946), (u'freedom', 17.21626037281668), (u'hire', 16.785266909588593), (u'walkman', 14.128590603937003), (u'volcano', 12.912195279612511), (u'mr', 12.476649250079015), (u'lasagna', 10.832753737932673), (u'milton', 9.419060402624668), (u'encyclopedia', 9.419060402624668), (u'berl', 9.419060402624668), (u'unlock', 8.60813018640834), (u'pull', 8.317766166719343), (u'key', 8.047189562170502), (u'lock', 7.783640596221253), (u'offic', 7.690286020676768)])\n",
      "\n",
      "(4, 16, [(u'joshua', 113.32853376224864), (u'emili', 41.44653167389283), (u'meat', 21.581057455185338), (u'bologna', 14.128590603937003), (u'gim', 12.824746787307683), (u'crave', 12.021999555697413), (u'parti', 10.986122886681098), (u'bra', 9.591581091193483), (u'soyburg', 9.419060402624668), (u'pastrami', 8.60813018640834), (u'takeoff', 8.014666370464942), (u'salami', 8.014666370464942), (u'glow', 8.014666370464942), (u'nauseou', 8.014666370464942), (u'twoweek', 8.014666370464942), (u'cherri', 8.014666370464942), (u'albino', 8.014666370464942), (u'surpris', 7.690286020676768), (u'l', 7.568379267836522), (u'album', 6.8679744089702925)])\n",
      "(4, 16, [(u'joshua', 113.32853376224864), (u'emili', 41.44653167389283), (u'meat', 21.581057455185338), (u'bologna', 14.128590603937003), (u'gim', 12.824746787307683), (u'crave', 12.021999555697413), (u'parti', 10.986122886681098), (u'bra', 9.591581091193483), (u'soyburg', 9.419060402624668), (u'pastrami', 8.60813018640834), (u'takeoff', 8.014666370464942), (u'salami', 8.014666370464942), (u'glow', 8.014666370464942), (u'nauseou', 8.014666370464942), (u'twoweek', 8.014666370464942), (u'cherri', 8.014666370464942), (u'albino', 8.014666370464942), (u'surpris', 7.690286020676768), (u'l', 7.568379267836522), (u'm', 7.568379267836522)])\n",
      "\n",
      "(5, 6, [(u'danni', 79.46798231228348), (u'emili', 52.95945713886306), (u'fog', 47.34471602524587), (u'yeti', 18.838120805249336), (u'coat', 14.55609079175885), (u'waffl', 10.832753737932673), (u'unreason', 10.30196161345544), (u'fur', 9.887510598012987), (u'huntley', 9.419060402624668), (u'heirloom', 9.419060402624668), (u'mink', 9.419060402624668), (u'henc', 9.419060402624668), (u'crematorium', 9.419060402624668), (u'judgement', 8.60813018640834), (u'furnitur', 8.12415060330663), (u'cremat', 8.014666370464942), (u'storag', 8.014666370464942), (u'chri', 7.221835825288449), (u'phil', 7.221835825288449), (u'bomb', 7.221835825288449)])\n",
      "(5, 6, [(u'danni', 79.46798231228348), (u'emili', 52.95945713886306), (u'fog', 47.34471602524587), (u'yeti', 18.838120805249336), (u'coat', 14.55609079175885), (u'waffl', 10.832753737932673), (u'unreason', 10.30196161345544), (u'fur', 9.887510598012987), (u'mink', 9.419060402624668), (u'huntley', 9.419060402624668), (u'heirloom', 9.419060402624668), (u'henc', 9.419060402624668), (u'crematorium', 9.419060402624668), (u'judgement', 8.60813018640834), (u'furnitur', 8.12415060330663), (u'cremat', 8.014666370464942), (u'storag', 8.014666370464942), (u'chri', 7.221835825288449), (u'phil', 7.221835825288449), (u'bomb', 7.221835825288449)])\n",
      "\n",
      "(5, 9, [(u'leedbett', 51.648781118450046), (u'sandwich', 39.55004239205195), (u'teacher', 28.214442932076903), (u'eyr', 23.54765100656167), (u'jane', 18.920948169591306), (u'cynthia', 18.838120805249336), (u'addict', 17.21626037281668), (u'video', 15.389696144769221), (u'heathcliff', 14.128590603937003), (u'underwear', 13.621371043387192), (u'dr', 13.183347464017316), (u'psychiatrist', 12.912195279612511), (u'disgust', 12.712215321391783), (u'explain', 12.542316284596385), (u'camera', 11.51292546497023), (u'read', 10.986122886681098), (u'ate', 10.75055681536833), (u'more', 9.419060402624668), (u'femin', 9.419060402624668), (u'cyborg', 9.419060402624668)])\n",
      "(5, 9, [(u'leedbett', 51.648781118450046), (u'sandwich', 39.55004239205195), (u'teacher', 28.214442932076903), (u'eyr', 23.54765100656167), (u'jane', 18.920948169591306), (u'cynthia', 18.838120805249336), (u'addict', 17.21626037281668), (u'video', 15.389696144769221), (u'heathcliff', 14.128590603937003), (u'underwear', 13.621371043387192), (u'dr', 13.183347464017316), (u'psychiatrist', 12.912195279612511), (u'disgust', 12.712215321391783), (u'explain', 12.542316284596385), (u'camera', 11.51292546497023), (u'read', 10.986122886681098), (u'ate', 10.75055681536833), (u'femin', 9.419060402624668), (u'cyborg', 9.419060402624668), (u'mental', 8.671115273688493)])\n",
      "\n",
      "(5, 16, [(u'cop', 49.136319884234794), (u'pivot', 23.54765100656167), (u'couch', 22.532130774077405), (u'salesman', 21.665507475865347), (u'badg', 21.665507475865347), (u'dream', 21.50111363073666), (u'gari', 20.23260230527315), (u'backup', 18.05458956322112), (u'saleswoman', 16.029332740929885), (u'smoke', 14.55609079175885), (u'crossword', 14.443671650576897), (u'sipowicz', 14.128590603937003), (u'arrest', 12.712215321391783), (u'298', 9.419060402624668), (u'nypd', 9.419060402624668), (u'sweatshirt', 9.419060402624668), (u'pivet', 9.419060402624668), (u'undercov', 9.419060402624668), (u'tree', 8.788898309344878), (u'polic', 8.671115273688493)])\n",
      "(5, 16, [(u'cop', 49.136319884234794), (u'pivot', 23.54765100656167), (u'couch', 22.532130774077405), (u'salesman', 21.665507475865347), (u'badg', 21.665507475865347), (u'dream', 21.50111363073666), (u'gari', 20.23260230527315), (u'backup', 18.05458956322112), (u'saleswoman', 16.029332740929885), (u'smoke', 14.55609079175885), (u'crossword', 14.443671650576897), (u'sipowicz', 14.128590603937003), (u'arrest', 12.712215321391783), (u'298', 9.419060402624668), (u'nypd', 9.419060402624668), (u'pivet', 9.419060402624668), (u'sweatshirt', 9.419060402624668), (u'undercov', 9.419060402624668), (u'tree', 8.788898309344878), (u'polic', 8.671115273688493)])\n",
      "\n",
      "(6, 9, [(u'dessert', 34.33987204485146), (u'geller', 31.859756371375184), (u'thanksgiv', 30.779392289538443), (u'janin', 26.366694928034633), (u'trifl', 25.824390559225023), (u'dancer', 22.24637681243562), (u'beef', 21.665507475865347), (u'mr', 20.101268236238415), (u'layer', 15.45521226679158), (u'custard', 12.912195279612511), (u'english', 12.712215321391783), (u'dad', 10.986122886681098), (u'stone', 10.30196161345544), (u'banana', 10.30196161345544), (u'pot', 9.534161491043838), (u'jacqu', 9.419060402624668), (u'truffl', 9.419060402624668), (u'cousteau', 9.419060402624668), (u'jam', 9.273127360074948), (u'jack', 9.210340371976184)])\n",
      "(6, 9, [(u'dessert', 34.33987204485146), (u'geller', 31.859756371375184), (u'thanksgiv', 30.779392289538443), (u'janin', 26.366694928034633), (u'trifl', 25.824390559225023), (u'dancer', 22.24637681243562), (u'beef', 21.665507475865347), (u'mr', 20.101268236238415), (u'layer', 15.45521226679158), (u'custard', 12.912195279612511), (u'english', 12.712215321391783), (u'dad', 10.986122886681098), (u'banana', 10.30196161345544), (u'stone', 10.30196161345544), (u'pot', 9.534161491043838), (u'jacqu', 9.419060402624668), (u'truffl', 9.419060402624668), (u'cousteau', 9.419060402624668), (u'jam', 9.273127360074948), (u'jack', 9.210340371976184)])\n",
      "\n",
      "(6, 10, [(u'janin', 75.80424791809958), (u'platform', 37.67624161049867), (u'director', 31.191623125197538), (u'eve', 22.705137803509565), (u'routin', 22.705137803509565), (u'present', 22.18070977791825), (u'danc', 18.021826694558577), (u'tall', 16.785266909588593), (u'millennium', 14.128590603937003), (u'christma', 13.621371043387192), (u'dick', 12.021999555697413), (u'clark', 11.352568901754783), (u'closet', 9.591581091193483), (u'dancer', 9.534161491043838), (u'azzz', 9.419060402624668), (u'co', 9.419060402624668), (u'snippi', 9.419060402624668), (u'hee', 9.419060402624668), (u'tape', 8.317766166719343), (u'countdown', 7.568379267836522)])\n",
      "(6, 10, [(u'janin', 75.80424791809958), (u'platform', 37.67624161049867), (u'director', 31.191623125197538), (u'eve', 22.705137803509565), (u'routin', 22.705137803509565), (u'present', 22.18070977791825), (u'danc', 18.021826694558577), (u'tall', 16.785266909588593), (u'millennium', 14.128590603937003), (u'christma', 13.621371043387192), (u'dick', 12.021999555697413), (u'clark', 11.352568901754783), (u'closet', 9.591581091193483), (u'dancer', 9.534161491043838), (u'azzz', 9.419060402624668), (u'co', 9.419060402624668), (u'hee', 9.419060402624668), (u'snippi', 9.419060402624668), (u'tape', 8.317766166719343), (u'countdown', 7.568379267836522)])\n",
      "\n",
      "(6, 12, [(u'pushov', 37.67624161049867), (u'joke', 37.352817814715735), (u'mainten', 32.05866548185977), (u'gunther', 23.56700413903814), (u'monkey', 20.519594859692294), (u'flaki', 18.838120805249336), (u'playboy', 18.05458956322112), (u'print', 15.890269151739728), (u'patron', 14.128590603937003), (u'easygo', 14.128590603937003), (u'custom', 13.195286648076292), (u'apron', 10.832753737932673), (u'argu', 10.30196161345544), (u'doctor', 9.887510598012987), (u'flexibl', 9.419060402624668), (u'dy', 9.419060402624668), (u'steve', 8.987196820661973), (u'coffeehous', 8.671115273688493), (u'journal', 8.60813018640834), (u'stole', 8.317766166719343)])\n",
      "(6, 12, [(u'pushov', 37.67624161049867), (u'joke', 37.352817814715735), (u'mainten', 32.05866548185977), (u'gunther', 23.56700413903814), (u'monkey', 20.519594859692294), (u'flaki', 18.838120805249336), (u'playboy', 18.05458956322112), (u'print', 15.890269151739728), (u'patron', 14.128590603937003), (u'easygo', 14.128590603937003), (u'custom', 13.195286648076292), (u'apron', 10.832753737932673), (u'argu', 10.30196161345544), (u'doctor', 9.887510598012987), (u'dy', 9.419060402624668), (u'flexibl', 9.419060402624668), (u'steve', 8.987196820661973), (u'coffeehous', 8.671115273688493), (u'journal', 8.60813018640834), (u'stole', 8.317766166719343)])\n",
      "\n",
      "(6, 20, [(u'mac', 18.920948169591306), (u'allergi', 18.838120805249336), (u'interview', 16.11809565095832), (u'chees', 13.183347464017316), (u'leedbett', 12.912195279612511), (u'forgiv', 12.824746787307683), (u'sandwich', 10.986122886681098), (u'lorrain', 8.60813018640834), (u'janic', 8.317766166719343), (u'youyouyouy', 8.014666370464942), (u'pacino', 8.014666370464942), (u'line', 7.690286020676768), (u'gooood', 7.568379267836522), (u'piti', 7.221835825288449), (u'al', 7.221835825288449), (u'fool', 7.193685818395112), (u'fri', 6.591673732008658), (u'board', 6.591673732008658), (u'robot', 6.182084906716632), (u'coulda', 6.182084906716632)])\n",
      "(6, 20, [(u'mac', 18.920948169591306), (u'allergi', 18.838120805249336), (u'interview', 16.11809565095832), (u'chees', 13.183347464017316), (u'leedbett', 12.912195279612511), (u'forgiv', 12.824746787307683), (u'sandwich', 10.986122886681098), (u'lorrain', 8.60813018640834), (u'janic', 8.317766166719343), (u'youyouyouy', 8.014666370464942), (u'pacino', 8.014666370464942), (u'line', 7.690286020676768), (u'gooood', 7.568379267836522), (u'piti', 7.221835825288449), (u'al', 7.221835825288449), (u'fool', 7.193685818395112), (u'fri', 6.591673732008658), (u'board', 6.591673732008658), (u'coulda', 6.182084906716632), (u'robot', 6.182084906716632)])\n",
      "\n",
      "(6, 23, [(u'paul', 154.5521226679158), (u'ring', 52.67918572255584), (u'jewel', 51.648781118450046), (u'male', 26.390573296152585), (u'custom', 18.47340130730681), (u'tiara', 12.912195279612511), (u'gumbal', 12.021999555697413), (u'knick', 10.83220080440884), (u'jail', 9.887510598012987), (u'stifl', 9.419060402624668), (u'1920', 9.419060402624668), (u'surfer', 9.419060402624668), (u'carat', 9.419060402624668), (u'chicken', 8.958797346140274), (u'brochur', 8.60813018640834), (u'ticket', 8.047189562170502), (u'musket', 8.014666370464942), (u'hug', 7.783640596221253), (u'engag', 7.783640596221253), (u'overwhelm', 7.221835825288449)])\n",
      "(6, 23, [(u'paul', 154.5521226679158), (u'ring', 52.67918572255584), (u'jewel', 51.648781118450046), (u'male', 26.390573296152585), (u'custom', 18.47340130730681), (u'tiara', 12.912195279612511), (u'gumbal', 12.021999555697413), (u'knick', 10.83220080440884), (u'jail', 9.887510598012987), (u'stifl', 9.419060402624668), (u'surfer', 9.419060402624668), (u'1920', 9.419060402624668), (u'carat', 9.419060402624668), (u'chicken', 8.958797346140274), (u'brochur', 8.60813018640834), (u'ticket', 8.047189562170502), (u'musket', 8.014666370464942), (u'hug', 7.783640596221253), (u'engag', 7.783640596221253), (u'overwhelm', 7.221835825288449)])\n",
      "\n",
      "(7, 1, [(u'thunder', 28.051332296627297), (u'19', 26.366694928034633), (u'whack', 17.21626037281668), (u'deposit', 16.029332740929885), (u'sup', 15.136758535673044), (u'sing', 13.621371043387192), (u'engag', 13.621371043387192), (u'playstat', 12.021999555697413), (u'especi', 11.989476363991853), (u'champagn', 11.561487031584658), (u'plaza', 11.352568901754783), (u'pari', 9.273127360074948), (u'wed', 9.010913347279288), (u'31', 8.014666370464942), (u'bonu', 7.221835825288449), (u'wisdom', 7.221835825288449), (u'steal', 7.16703787691222), (u'gay', 7.16703787691222), (u'sixteen', 6.8679744089702925), (u'stolen', 6.8679744089702925)])\n",
      "(7, 1, [(u'thunder', 28.051332296627297), (u'19', 26.366694928034633), (u'whack', 17.21626037281668), (u'deposit', 16.029332740929885), (u'sup', 15.136758535673044), (u'sing', 13.621371043387192), (u'engag', 13.621371043387192), (u'playstat', 12.021999555697413), (u'especi', 11.989476363991853), (u'champagn', 11.561487031584658), (u'plaza', 11.352568901754783), (u'pari', 9.273127360074948), (u'wed', 9.010913347279288), (u'31', 8.014666370464942), (u'bonu', 7.221835825288449), (u'wisdom', 7.221835825288449), (u'steal', 7.16703787691222), (u'gay', 7.16703787691222), (u'stolen', 6.8679744089702925), (u'sixteen', 6.8679744089702925)])\n",
      "\n",
      "(7, 15, [(u'cecilia', 240.18604026692904), (u'jessica', 46.94193286437492), (u'dina', 36.06599866709224), (u'lockhart', 32.96671140918634), (u'own', 29.662531794038962), (u'scottish', 25.824390559225023), (u'tom', 24.72833962686653), (u'charact', 16.785266909588593), (u'cell', 16.479184330021646), (u'slap', 15.45521226679158), (u'brain', 14.909439898728003), (u'dial', 14.16606672028108), (u'dool', 14.128590603937003), (u'monro', 14.128590603937003), (u'eee', 14.128590603937003), (u'barney', 12.021999555697413), (u'transplant', 12.021999555697413), (u'speed', 11.561487031584658), (u'phone', 11.090354888959125), (u'fan', 10.986122886681098)])\n",
      "(7, 15, [(u'cecilia', 240.18604026692904), (u'jessica', 46.94193286437492), (u'dina', 36.06599866709224), (u'lockhart', 32.96671140918634), (u'scottish', 25.824390559225023), (u'tom', 24.72833962686653), (u'charact', 16.785266909588593), (u'cell', 16.479184330021646), (u'slap', 15.45521226679158), (u'brain', 14.909439898728003), (u'dial', 14.16606672028108), (u'dool', 14.128590603937003), (u'monro', 14.128590603937003), (u'eee', 14.128590603937003), (u'barney', 12.021999555697413), (u'transplant', 12.021999555697413), (u'speed', 11.561487031584658), (u'phone', 11.090354888959125), (u'fan', 10.986122886681098), (u'coma', 10.832753737932673)])\n",
      "\n",
      "(7, 19, [(u'cassi', 103.29756223690009), (u'director', 37.42994775023704), (u'cast', 27.333973147668004), (u'nuditi', 12.021999555697413), (u'estel', 11.561487031584658), (u'shower', 11.266065387038703), (u'jewish', 10.832753737932673), (u'cousin', 10.259797429846147), (u'movi', 9.887510598012987), (u'option', 9.887510598012987), (u'toothpick', 9.419060402624668), (u'marilyn', 9.419060402624668), (u'napl', 9.419060402624668), (u'scorces', 9.419060402624668), (u'authent', 8.60813018640834), (u'shrill', 8.60813018640834), (u'bridal', 8.60813018640834), (u'cathol', 8.60813018640834), (u'scene', 8.047189562170502), (u'martin', 8.014666370464942)])\n",
      "(7, 19, [(u'cassi', 103.29756223690009), (u'director', 37.42994775023704), (u'cast', 27.333973147668004), (u'nuditi', 12.021999555697413), (u'estel', 11.561487031584658), (u'shower', 11.266065387038703), (u'jewish', 10.832753737932673), (u'cousin', 10.259797429846147), (u'movi', 9.887510598012987), (u'option', 9.887510598012987), (u'toothpick', 9.419060402624668), (u'marilyn', 9.419060402624668), (u'napl', 9.419060402624668), (u'scorces', 9.419060402624668), (u'authent', 8.60813018640834), (u'cathol', 8.60813018640834), (u'shrill', 8.60813018640834), (u'bridal', 8.60813018640834), (u'scene', 8.047189562170502), (u'martin', 8.014666370464942)])\n",
      "\n",
      "(7, 20, [(u'melissa', 146.3382131689418), (u'batman', 47.34471602524587), (u'tux', 41.20784645382176), (u'007', 32.96671140918634), (u'rayray', 23.54765100656167), (u'tuxedo', 16.029332740929885), (u'jame', 14.978661367769954), (u'kilmer', 14.128590603937003), (u'val', 14.128590603937003), (u'parent', 13.183347464017316), (u'espn', 12.912195279612511), (u'bond', 12.824746787307683), (u'pin', 11.561487031584658), (u'coconut', 10.832753737932673), (u'comin', 10.259797429846147), (u'invit', 9.704060527839234), (u'shut', 9.656627474604601), (u'doubleoh', 9.419060402624668), (u'sangria', 9.419060402624668), (u'luau', 9.419060402624668)])\n",
      "(7, 20, [(u'melissa', 146.3382131689418), (u'batman', 47.34471602524587), (u'tux', 41.20784645382176), (u'007', 32.96671140918634), (u'rayray', 23.54765100656167), (u'tuxedo', 16.029332740929885), (u'jame', 14.978661367769954), (u'kilmer', 14.128590603937003), (u'val', 14.128590603937003), (u'parent', 13.183347464017316), (u'espn', 12.912195279612511), (u'bond', 12.824746787307683), (u'pin', 11.561487031584658), (u'coconut', 10.832753737932673), (u'comin', 10.259797429846147), (u'invit', 9.704060527839234), (u'shut', 9.656627474604601), (u'doubleoh', 9.419060402624668), (u'sangria', 9.419060402624668), (u'sigma', 9.419060402624668)])\n",
      "\n",
      "(7, 21, [(u'vow', 30.273517071346088), (u'volvo', 12.912195279612511), (u'40', 10.30196161345544), (u'write', 9.704060527839234), (u'chicken', 8.958797346140274), (u'246', 8.60813018640834), (u'dealership', 8.60813018640834), (u'inappropri', 7.694848072384611), (u'lotion', 7.221835825288449), (u'hypothet', 7.221835825288449), (u'gang', 6.8679744089702925), (u'sooner', 5.991464547107982), (u'youreyour', 4.795790545596741), (u'taught', 4.795790545596741), (u'4', 4.795790545596741), (u'gasp', 4.709530201312334), (u'cokediet', 4.709530201312334), (u'london', 4.394449154672439), (u'holocaust', 4.30406509320417), (u'erogen', 4.30406509320417)])\n",
      "(7, 21, [(u'vow', 30.273517071346088), (u'volvo', 12.912195279612511), (u'40', 10.30196161345544), (u'write', 9.704060527839234), (u'chicken', 8.958797346140274), (u'246', 8.60813018640834), (u'dealership', 8.60813018640834), (u'inappropri', 7.694848072384611), (u'lotion', 7.221835825288449), (u'hypothet', 7.221835825288449), (u'gang', 6.8679744089702925), (u'sooner', 5.991464547107982), (u'youreyour', 4.795790545596741), (u'taught', 4.795790545596741), (u'4', 4.795790545596741), (u'gasp', 4.709530201312334), (u'cokediet', 4.709530201312334), (u'london', 4.394449154672439), (u'erogen', 4.30406509320417), (u'willwil', 4.30406509320417)])\n",
      "\n",
      "(7, 22, [(u'policeman', 92.16866326034683), (u'helena', 89.48107382493434), (u'jake', 34.43252074563336), (u'porsch', 32.95836866004329), (u'driver', 30.905884840366316), (u'drive', 21.50111363073666), (u'race', 18.920948169591306), (u'ignit', 18.838120805249336), (u'licens', 16.479184330021646), (u'car', 15.249237972318797), (u'vega', 14.451858789480823), (u'wheel', 14.443671650576897), (u'girabaldi', 14.128590603937003), (u'steer', 13.735948817940585), (u'silk', 12.912195279612511), (u'drag', 11.332853376224865), (u'underwear', 9.729550745276565), (u'bakersfield', 9.419060402624668), (u'leman', 9.419060402624668), (u'sagittariu', 9.419060402624668)])\n",
      "(7, 22, [(u'policeman', 92.16866326034683), (u'helena', 89.48107382493434), (u'jake', 34.43252074563336), (u'porsch', 32.95836866004329), (u'driver', 30.905884840366316), (u'drive', 21.50111363073666), (u'race', 18.920948169591306), (u'ignit', 18.838120805249336), (u'licens', 16.479184330021646), (u'car', 15.249237972318797), (u'vega', 14.451858789480823), (u'wheel', 14.443671650576897), (u'girabaldi', 14.128590603937003), (u'steer', 13.735948817940585), (u'silk', 12.912195279612511), (u'drag', 11.332853376224865), (u'underwear', 9.729550745276565), (u'leman', 9.419060402624668), (u'bakersfield', 9.419060402624668), (u'sagittariu', 9.419060402624668)])\n",
      "\n",
      "(8, 5, [(u'tim', 102.17312011579305), (u'kash', 84.77154362362201), (u'tobi', 51.648781118450046), (u'bob', 50.14208926268991), (u'franklin', 38.736585838837534), (u'mona', 24.72833962686653), (u'costar', 22.705137803509565), (u'pesto', 18.838120805249336), (u'muriel', 17.21626037281668), (u'ukrainian', 17.21626037281668), (u'fire', 15.249237972318797), (u'ferri', 15.136758535673044), (u'sou', 14.128590603937003), (u'ramoray', 12.912195279612511), (u'film', 11.561487031584658), (u'pastri', 11.352568901754783), (u'drake', 11.332853376224865), (u'donut', 10.832753737932673), (u'jessica', 10.832753737932673), (u'chef', 9.729550745276565)])\n",
      "(8, 5, [(u'tim', 102.17312011579305), (u'kash', 84.77154362362201), (u'tobi', 51.648781118450046), (u'bob', 50.14208926268991), (u'franklin', 38.736585838837534), (u'mona', 24.72833962686653), (u'costar', 22.705137803509565), (u'pesto', 18.838120805249336), (u'muriel', 17.21626037281668), (u'ukrainian', 17.21626037281668), (u'fire', 15.249237972318797), (u'ferri', 15.136758535673044), (u'sou', 14.128590603937003), (u'ramoray', 12.912195279612511), (u'film', 11.561487031584658), (u'pastri', 11.352568901754783), (u'm', 11.352568901754783), (u'drake', 11.332853376224865), (u'donut', 10.832753737932673), (u'jessica', 10.832753737932673)])\n",
      "\n",
      "(8, 8, [(u'mona', 80.36710378731621), (u'hooker', 47.67080745521918), (u'green', 41.58883083359672), (u'stripper', 41.31469979452329), (u'bachelor', 32.49826121379802), (u'dr', 31.859756371375184), (u'stu', 30.128455652429192), (u'74', 28.257181207874005), (u'bachelorett', 16.029332740929885), (u'parti', 15.380572041353537), (u'daddi', 9.729550745276565), (u'latour', 9.419060402624668), (u'tivo', 9.419060402624668), (u'unti', 9.419060402624668), (u'lafit', 9.419060402624668), (u'sewag', 9.419060402624668), (u'waiter', 8.788898309344878), (u'tramp', 8.60813018640834), (u'grandchild', 8.60813018640834), (u'smoke', 8.317766166719343)])\n",
      "(8, 8, [(u'mona', 80.36710378731621), (u'hooker', 47.67080745521918), (u'green', 41.58883083359672), (u'stripper', 41.31469979452329), (u'bachelor', 32.49826121379802), (u'dr', 31.859756371375184), (u'stu', 30.128455652429192), (u'74', 28.257181207874005), (u'bachelorett', 16.029332740929885), (u'parti', 15.380572041353537), (u'daddi', 9.729550745276565), (u'tivo', 9.419060402624668), (u'unti', 9.419060402624668), (u'lafit', 9.419060402624668), (u'latour', 9.419060402624668), (u'sewag', 9.419060402624668), (u'waiter', 8.788898309344878), (u'grandchild', 8.60813018640834), (u'tramp', 8.60813018640834), (u'smoke', 8.317766166719343)])\n",
      "\n",
      "(8, 14, [(u'massag', 17.51319134149782), (u'closet', 16.785266909588593), (u'contract', 16.479184330021646), (u'pickl', 14.451858789480823), (u'mild', 14.128590603937003), (u'discomfort', 12.912195279612511), (u'kick', 11.090354888959125), (u'braxtonhick', 9.419060402624668), (u'alexandra', 9.419060402624668), (u'wolf', 9.419060402624668), (u'hamm', 9.419060402624668), (u'mia', 8.60813018640834), (u'richard', 8.317766166719343), (u'hospit', 8.317766166719343), (u'tuna', 7.568379267836522), (u'feminin', 7.568379267836522), (u'push', 7.16703787691222), (u'nois', 6.907755278982138), (u'bobbi', 6.3561076606958915), (u'uteru', 6.182084906716632)])\n",
      "(8, 14, [(u'massag', 17.51319134149782), (u'closet', 16.785266909588593), (u'contract', 16.479184330021646), (u'pickl', 14.451858789480823), (u'mild', 14.128590603937003), (u'discomfort', 12.912195279612511), (u'kick', 11.090354888959125), (u'hamm', 9.419060402624668), (u'braxtonhick', 9.419060402624668), (u'alexandra', 9.419060402624668), (u'wolf', 9.419060402624668), (u'mia', 8.60813018640834), (u'richard', 8.317766166719343), (u'hospit', 8.317766166719343), (u'tuna', 7.568379267836522), (u'feminin', 7.568379267836522), (u'push', 7.16703787691222), (u'nois', 6.907755278982138), (u'bobbi', 6.3561076606958915), (u'uteru', 6.182084906716632)])\n",
      "\n",
      "(8, 16, [(u'soul', 37.84189633918261), (u'mate', 34.33987204485146), (u'chees', 26.366694928034633), (u'waiter', 19.775021196025975), (u'bread', 9.534161491043838), (u'hobb', 9.419060402624668), (u'destin', 9.419060402624668), (u'tomato', 8.987196820661973), (u'sundri', 8.60813018640834), (u'rossross', 7.568379267836522), (u'hesh', 7.221835825288449), (u'tall', 7.193685818395112), (u'vermont', 6.8679744089702925), (u'smelli', 6.8679744089702925), (u'hous', 5.493061443340549), (u'2', 5.375278407684165), (u'ummnow', 4.709530201312334), (u'uhlook', 4.709530201312334), (u'brows', 4.709530201312334), (u'villa', 4.709530201312334)])\n",
      "(8, 16, [(u'don', 76.13933051941696), (u'soul', 37.84189633918261), (u'mate', 34.33987204485146), (u'chees', 26.366694928034633), (u'waiter', 19.775021196025975), (u'bread', 9.534161491043838), (u'hobb', 9.419060402624668), (u'destin', 9.419060402624668), (u'tomato', 8.987196820661973), (u'sundri', 8.60813018640834), (u'rossross', 7.568379267836522), (u'hesh', 7.221835825288449), (u'tall', 7.193685818395112), (u'vermont', 6.8679744089702925), (u'smelli', 6.8679744089702925), (u'hous', 5.493061443340549), (u'2', 5.375278407684165), (u'ummnow', 4.709530201312334), (u'uhlook', 4.709530201312334), (u'brows', 4.709530201312334)])\n",
      "\n",
      "(8, 20, [(u'green', 58.224363167035406), (u'bamboozl', 37.67624161049867), (u'ray', 34.05770670526435), (u'mr', 31.191623125197538), (u'wick', 28.257181207874005), (u'wango', 28.257181207874005), (u'wheel', 25.27642538850957), (u'card', 24.141568686511505), (u'spin', 15.45521226679158), (u'mayhem', 14.128590603937003), (u'madr', 14.128590603937003), (u'googl', 14.128590603937003), (u'mother', 13.183347464017316), (u'geni', 12.912195279612511), (u'correct', 12.824746787307683), (u'nanni', 12.021999555697413), (u'erin', 12.021999555697413), (u'bonu', 10.832753737932673), (u'fear', 10.556229318461034), (u'contest', 9.939626599152001)])\n",
      "(8, 20, [(u'green', 58.224363167035406), (u'bamboozl', 37.67624161049867), (u'ray', 34.05770670526435), (u'mr', 31.191623125197538), (u'wick', 28.257181207874005), (u'wango', 28.257181207874005), (u'wheel', 25.27642538850957), (u'card', 24.141568686511505), (u'spin', 15.45521226679158), (u'mayhem', 14.128590603937003), (u'googl', 14.128590603937003), (u'madr', 14.128590603937003), (u'mother', 13.183347464017316), (u'geni', 12.912195279612511), (u'correct', 12.824746787307683), (u'nanni', 12.021999555697413), (u'erin', 12.021999555697413), (u'bonu', 10.832753737932673), (u'fear', 10.556229318461034), (u'contest', 9.939626599152001)])\n",
      "\n",
      "(9, 1, [(u'propos', 69.07755278982138), (u'ring', 24.95329850015803), (u'geller', 24.169470350698415), (u'engag', 19.45910149055313), (u'mr', 15.942385152878742), (u'nurs', 14.909439898728003), (u'breast', 10.75055681536833), (u'knee', 10.556229318461034), (u'dad', 9.887510598012987), (u'granddaught', 9.419060402624668), (u'liddi', 9.419060402624668), (u'peek', 8.60813018640834), (u'ovul', 8.014666370464942), (u'emma', 7.783640596221253), (u'master', 7.568379267836522), (u'posit', 7.193685818395112), (u'puppet', 6.8679744089702925), (u'conceiv', 6.8679744089702925), (u'rosss', 6.437751649736401), (u'gentl', 6.3561076606958915)])\n",
      "(9, 1, [(u'propos', 69.07755278982138), (u'ring', 24.95329850015803), (u'geller', 24.169470350698415), (u'engag', 19.45910149055313), (u'mr', 15.942385152878742), (u'nurs', 14.909439898728003), (u'breast', 10.75055681536833), (u'knee', 10.556229318461034), (u'dad', 9.887510598012987), (u'granddaught', 9.419060402624668), (u'liddi', 9.419060402624668), (u'peek', 8.60813018640834), (u'ovul', 8.014666370464942), (u'emma', 7.783640596221253), (u'master', 7.568379267836522), (u'posit', 7.193685818395112), (u'conceiv', 6.8679744089702925), (u'puppet', 6.8679744089702925), (u'rosss', 6.437751649736401), (u'gentl', 6.3561076606958915)])\n",
      "\n",
      "(9, 9, [(u'mike', 83.67413640937846), (u'tulsa', 19.068322982087675), (u'beer', 17.51319134149782), (u'courtsid', 17.21626037281668), (u'seduct', 14.128590603937003), (u'eyecontact', 14.128590603937003), (u'lager', 14.128590603937003), (u'escap', 12.364169813433264), (u'bill', 10.986122886681098), (u'um', 9.010913347279288), (u'art', 8.987196820661973), (u'jo', 8.60813018640834), (u'keyboard', 8.014666370464942), (u'lynn', 8.014666370464942), (u'practic', 7.783640596221253), (u'cheat', 7.694848072384611), (u'kevin', 7.568379267836522), (u'suitcas', 7.568379267836522), (u'piano', 6.591673732008658), (u'scene', 6.437751649736401)])\n",
      "(9, 9, [(u'mike', 83.67413640937846), (u'tulsa', 19.068322982087675), (u'beer', 17.51319134149782), (u'courtsid', 17.21626037281668), (u'seduct', 14.128590603937003), (u'lager', 14.128590603937003), (u'eyecontact', 14.128590603937003), (u'escap', 12.364169813433264), (u'bill', 10.986122886681098), (u'um', 9.010913347279288), (u'art', 8.987196820661973), (u'jo', 8.60813018640834), (u'keyboard', 8.014666370464942), (u'lynn', 8.014666370464942), (u'practic', 7.783640596221253), (u'cheat', 7.694848072384611), (u'kevin', 7.568379267836522), (u'suitcas', 7.568379267836522), (u'piano', 6.591673732008658), (u'scene', 6.437751649736401)])\n",
      "\n",
      "(9, 10, [(u'wendi', 45.56001018316904), (u'christma', 42.81002327921689), (u'merri', 25.424430642783566), (u'santa', 20.519594859692294), (u'ham', 20.036665926162357), (u'eve', 18.920948169591306), (u'present', 15.249237972318797), (u'oklahoma', 15.136758535673044), (u'armadillo', 12.912195279612511), (u'dictionari', 12.021999555697413), (u'holiday', 11.989476363991853), (u'goof', 11.352568901754783), (u'ben', 11.266065387038703), (u'snow', 9.273127360074948), (u'other', 9.273127360074948), (u'ho', 8.987196820661973), (u'ballet', 8.60813018640834), (u'ken', 8.014666370464942), (u'faster', 7.694848072384611), (u'hanukkah', 7.568379267836522)])\n",
      "(9, 10, [(u'wendi', 45.56001018316904), (u'christma', 42.81002327921689), (u'merri', 25.424430642783566), (u'santa', 20.519594859692294), (u'ham', 20.036665926162357), (u'eve', 18.920948169591306), (u'present', 15.249237972318797), (u'oklahoma', 15.136758535673044), (u'armadillo', 12.912195279612511), (u'dictionari', 12.021999555697413), (u'holiday', 11.989476363991853), (u'goof', 11.352568901754783), (u'ben', 11.266065387038703), (u'snow', 9.273127360074948), (u'ho', 8.987196820661973), (u'ballet', 8.60813018640834), (u'ken', 8.014666370464942), (u'faster', 7.694848072384611), (u'hanukkah', 7.568379267836522), (u'happili', 7.568379267836522)])\n",
      "\n",
      "(9, 18, [(u'lotteri', 64.56097639806255), (u'ticket', 33.7981961611161), (u'win', 31.884770305757485), (u'powerbal', 28.257181207874005), (u'psychic', 20.036665926162357), (u'gleba', 18.838120805249336), (u'wishbon', 18.838120805249336), (u'ok', 18.676408907357867), (u'charli', 16.9992800643373), (u'boohaki', 14.128590603937003), (u'earhart', 14.128590603937003), (u'emma', 13.621371043387192), (u'nightstand', 12.912195279612511), (u'casserol', 12.021999555697413), (u'mum', 12.021999555697413), (u'steve', 11.982929094215963), (u'tow', 11.352568901754783), (u'odd', 11.332853376224865), (u'knick', 10.83220080440884), (u'mommi', 10.556229318461034)])\n",
      "(9, 18, [(u'lotteri', 64.56097639806255), (u'ticket', 33.7981961611161), (u'win', 31.884770305757485), (u'powerbal', 28.257181207874005), (u'psychic', 20.036665926162357), (u'won', 19.45910149055313), (u'gleba', 18.838120805249336), (u'wishbon', 18.838120805249336), (u'ok', 18.676408907357867), (u'charli', 16.9992800643373), (u'boohaki', 14.128590603937003), (u'earhart', 14.128590603937003), (u'emma', 13.621371043387192), (u'nightstand', 12.912195279612511), (u'casserol', 12.021999555697413), (u'mum', 12.021999555697413), (u'steve', 11.982929094215963), (u'tow', 11.352568901754783), (u'odd', 11.332853376224865), (u'knick', 10.83220080440884)])\n",
      "\n",
      "(10, 2, [(u'frank', 93.49604035385514), (u'jr', 83.88050365951175), (u'owen', 68.86504149126672), (u'colleen', 56.51436241574801), (u'adopt', 56.029226722073595), (u'charli', 53.83105353706811), (u'fajita', 23.54765100656167), (u'bill', 15.380572041353537), (u'triplet', 14.978661367769954), (u'margarita', 10.832753737932673), (u'batch', 10.832753737932673), (u'lesli', 9.534161491043838), (u'carriag', 9.419060402624668), (u'frog', 8.60813018640834), (u'flan', 8.60813018640834), (u'intuit', 8.60813018640834), (u'alright', 7.690286020676768), (u'barbado', 7.221835825288449), (u'nut', 6.907755278982138), (u'alic', 6.3561076606958915)])\n",
      "(10, 2, [(u'frank', 93.49604035385514), (u'jr', 83.88050365951175), (u'owen', 68.86504149126672), (u'colleen', 56.51436241574801), (u'adopt', 56.029226722073595), (u'charli', 53.83105353706811), (u'fajita', 23.54765100656167), (u'bill', 15.380572041353537), (u'triplet', 14.978661367769954), (u'margarita', 10.832753737932673), (u'batch', 10.832753737932673), (u'lesli', 9.534161491043838), (u'carriag', 9.419060402624668), (u'flan', 8.60813018640834), (u'frog', 8.60813018640834), (u'intuit', 8.60813018640834), (u'alright', 7.690286020676768), (u'barbado', 7.221835825288449), (u'nut', 6.907755278982138), (u'alic', 6.3561076606958915)])\n",
      "\n",
      "(10, 6, [(u'benjamin', 176.46666882137097), (u'charli', 96.32925369791135), (u'grant', 30.91042453358316), (u'gladi', 28.257181207874005), (u'benji', 23.54765100656167), (u'li', 21.52032546602085), (u'lie', 19.775021196025975), (u'bieli', 18.838120805249336), (u'ok', 15.380572041353537), (u'tape', 14.55609079175885), (u'dr', 13.183347464017316), (u'hobart', 12.912195279612511), (u'exboyfriend', 12.364169813433264), (u'paleontolog', 11.561487031584658), (u'field', 11.561487031584658), (u'boss', 9.656627474604601), (u'chameleon', 9.419060402624668), (u'boscodictiasaur', 9.419060402624668), (u'likewis', 9.419060402624668), (u'excav', 9.419060402624668)])\n",
      "(10, 6, [(u'benjamin', 176.46666882137097), (u'charli', 96.32925369791135), (u'grant', 30.91042453358316), (u'gladi', 28.257181207874005), (u'benji', 23.54765100656167), (u'li', 21.52032546602085), (u'lie', 19.775021196025975), (u'bieli', 18.838120805249336), (u'ok', 15.380572041353537), (u'tape', 14.55609079175885), (u'dr', 13.183347464017316), (u'hobart', 12.912195279612511), (u'exboyfriend', 12.364169813433264), (u'paleontolog', 11.561487031584658), (u'field', 11.561487031584658), (u'boss', 9.656627474604601), (u'chameleon', 9.419060402624668), (u'likewis', 9.419060402624668), (u'boscodictiasaur', 9.419060402624668), (u'excav', 9.419060402624668)])\n",
      "\n",
      "(10, 12, [(u'mike', 103.1332378999316), (u'groomsman', 37.67624161049867), (u'bridesmaid', 24.043999111394825), (u'chappi', 23.54765100656167), (u'ok', 20.873633484694086), (u'choos', 20.519594859692294), (u'rehears', 18.546254720149896), (u'uhm', 18.420680743952367), (u'wed', 16.635532333438686), (u'snow', 15.45521226679158), (u'lorkin', 14.128590603937003), (u'groomsmen', 14.128590603937003), (u'corsag', 14.128590603937003), (u'marjori', 12.912195279612511), (u'sculptur', 12.912195279612511), (u'strike', 9.887510598012987), (u'minist', 9.534161491043838), (u'harpist', 9.419060402624668), (u'isl', 9.419060402624668), (u'1800', 9.419060402624668)])\n",
      "(10, 12, [(u'mike', 103.1332378999316), (u'groomsman', 37.67624161049867), (u'bridesmaid', 24.043999111394825), (u'chappi', 23.54765100656167), (u'ok', 20.873633484694086), (u'choos', 20.519594859692294), (u'rehears', 18.546254720149896), (u'uhm', 18.420680743952367), (u'wed', 16.635532333438686), (u'snow', 15.45521226679158), (u'lorkin', 14.128590603937003), (u'corsag', 14.128590603937003), (u'groomsmen', 14.128590603937003), (u'marjori', 12.912195279612511), (u'sculptur', 12.912195279612511), (u'strike', 9.887510598012987), (u'minist', 9.534161491043838), (u'harpist', 9.419060402624668), (u'isl', 9.419060402624668), (u'1800', 9.419060402624668)])\n",
      "\n",
      "(10, 13, [(u'erica', 84.1539968898819), (u'je', 64.56097639806255), (u'ok', 30.761144082707073), (u'claud', 28.257181207874005), (u'mappel', 23.54765100656167), (u'french', 21.581057455185338), (u'pooh', 14.128590603937003), (u'green', 13.862943611198906), (u'uhm', 13.815510557964275), (u'est', 12.912195279612511), (u'director', 12.476649250079014), (u'attack', 12.364169813433264), (u'le', 12.021999555697413), (u'un', 12.021999555697413), (u'la', 10.259797429846147), (u'father', 9.887510598012987), (u'speak', 9.656627474604601), (u'cest', 9.419060402624668), (u'rsum', 9.419060402624668), (u'alor', 9.419060402624668)])\n",
      "(10, 13, [(u'erica', 84.1539968898819), (u'je', 64.56097639806255), (u'ok', 30.761144082707073), (u'claud', 28.257181207874005), (u'mappel', 23.54765100656167), (u'french', 21.581057455185338), (u'pooh', 14.128590603937003), (u'green', 13.862943611198906), (u'uhm', 13.815510557964275), (u'est', 12.912195279612511), (u'director', 12.476649250079014), (u'attack', 12.364169813433264), (u'le', 12.021999555697413), (u'un', 12.021999555697413), (u'la', 10.259797429846147), (u'father', 9.887510598012987), (u'speak', 9.656627474604601), (u'cest', 9.419060402624668), (u'sedat', 9.419060402624668), (u'rsum', 9.419060402624668)])\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for x, y in zip(tfidf_w_stopwords, tfidf_wout_stopwords):\n",
    "    if x != y:\n",
    "        print(x)\n",
    "        print(y)\n",
    "        print"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Good! Now just quickly do the same w/ stopwords for 2,3-grams:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from nltk.util import ngrams\n",
    "\n",
    "bigrammed_data_w_stopwords = [(s, e, list(ngrams(d, 2))) for s, e, d in stemmed_data_w_stopwords]\n",
    "trigrammed_data_w_stopwords = [(s, e, list(ngrams(d, 3))) for s, e, d in stemmed_data_w_stopwords]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Season 1, episode 1: (u'cut', u'cut'): 68.865, (u'wine', u'guy'): 23.548, (u'grab', u'spoon'): 18.838, (u'push', u'stair'): 18.838, (u'paul', u'wine'): 18.838, (u'your', u'shoe'): 17.216, (u'know', u'paul'): 17.216, (u'paul', u'monica'): 17.216, (u'one', u'woman'): 14.444, (u'fifth', u'date'): 14.129, (u'shoe', u'your'): 12.912, (u'real', u'date'): 12.912, (u'la', u'vega'): 10.833, (u'woman', u'that'):  9.419, (u'bookcas', u'chandler'):  9.419, (u'great', u'butt'):  9.419, (u'stair', u'push'):  9.419, (u'ross', u'grab'):  9.419, (u'made', u'coffe'):  9.419, (u'morn', u'paul'):  9.419\n",
      "\n",
      "Season 1, episode 2: (u'mr', u'geller'): 31.192, (u'ew', u'ew'): 28.051, (u'dr', u'oberman'): 17.216, (u'ross', u'marsha'): 14.129, (u'barri', u'yeah'): 12.022, (u'carol', u'ross'): 11.561, (u'susan', u'ross'): 10.302, (u'chaotic', u'twirli'):  9.419, (u'helen', u'geller'):  9.419, (u'give', u'barri'):  9.419, (u'good', u'shake'):  9.419, (u'um', u'how'):  9.419, (u'alreadi', u'fluf'):  9.419, (u'na', u'helen'):  9.419, (u'job', u'done'):  9.419, (u'barri', u'went'):  9.419, (u'thank', u'give'):  9.419, (u'your', u'twin'):  9.419, (u'barri', u'back'):  9.419, (u'barri', u'oh'):  8.608\n",
      "\n",
      "Season 1, episode 3: (u'feel', u'thing'): 14.444, (u'wouldnt', u'fair'): 14.129, (u'alan', u'alan'): 14.129, (u'footbal', u'phone'): 14.129, (u'alan', u'ross'): 14.129, (u'like', u'paula'): 14.129, (u'notnotmin', u'notnotmin'): 14.129, (u'im', u'smoke'): 12.912, (u'thousand', u'dollar'): 12.364, (u'rest', u'poptart'):  9.419, (u'seven', u'thousand'):  9.419, (u'okay', u'lizzi'):  9.419, (u'alan', u'yeah'):  9.419, (u'friend', u'realli'):  9.419, (u'chi', u'chi'):  9.419, (u'live', u'farm'):  9.419, (u'paula', u'well'):  9.419, (u'futur', u'boyfriend'):  9.419, (u'hi', u'alan'):  9.419, (u'fine', u'lizzi'):  9.419\n",
      "\n",
      "Season 1, episode 4: (u'pizza', u'guy'): 32.059, (u'she', u'walk'): 12.912, (u'im', u'okay'): 10.397, (u'kill', u'monica'):  9.534, (u'drop', u'towel'):  9.419, (u'unusu', u'activ'):  9.419, (u'ice', u'night'):  9.419, (u'visa', u'card'):  9.419, (u'peach', u'pit'):  9.419, (u'live', u'villag'):  9.419, (u'ross', u'peach'):  9.419, (u'first', u'pay'):  9.419, (u'complet', u'satisfi'):  9.419, (u'see', u'lem'):  9.419, (u'omnipot', u'joey'):  9.419, (u'uh', u'omnipot'):  9.419, (u'walk', u'she'):  9.419, (u'kiki', u'well'):  9.419, (u'magic', u'bean'):  9.419, (u'night', u'first'):  9.419\n",
      "\n",
      "Season 1, episode 5: (u'la', u'la'): 22.705, (u'bob', u'joey'): 14.129, (u'hey', u'hey'):  9.888, (u'actor', u'bob'):  9.419, (u'he', u'sophist'):  9.419, (u'bob', u'monica'):  9.419, (u'night', u'saturday'):  9.419, (u'boom', u'boom'):  9.419, (u'real', u'job'):  9.419, (u'three', u'audit'):  9.419, (u'machin', u'woman'):  9.419, (u'audit', u'month'):  9.419, (u'angela', u'yeah'):  9.419, (u'laundri', u'done'):  9.419, (u'break', u'toni'):  9.419, (u'joey', u'angela'):  9.419, (u'basket', u'top'):  9.419, (u'sud', u'save'):  9.419, (u'red', u'sock'):  9.419, (u'sophist', u'real'):  9.419\n",
      "\n"
     ]
    }
   ],
   "source": [
    "bigram_tfidf_data = calculate_tfidf(bigrammed_data_w_stopwords, count_doc_frequencies(bigrammed_data_w_stopwords))\n",
    "print_tfidf_data(bigram_tfidf_data[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" cellpadding=\"3\" cellspacing=\"0\"  style=\"border:1px solid black;border-collapse:collapse;\"><tr><td>(u'cut',&nbspu'cut')</td><td>68.8650</td></tr><tr><td>(u'wine',&nbspu'guy')</td><td>23.5477</td></tr><tr><td>(u'grab',&nbspu'spoon')</td><td>18.8381</td></tr><tr><td>(u'push',&nbspu'stair')</td><td>18.8381</td></tr><tr><td>(u'paul',&nbspu'wine')</td><td>18.8381</td></tr><tr><td>(u'your',&nbspu'shoe')</td><td>17.2163</td></tr><tr><td>(u'know',&nbspu'paul')</td><td>17.2163</td></tr><tr><td>(u'paul',&nbspu'monica')</td><td>17.2163</td></tr><tr><td>(u'one',&nbspu'woman')</td><td>14.4437</td></tr><tr><td>(u'fifth',&nbspu'date')</td><td>14.1286</td></tr><tr><td>(u'shoe',&nbspu'your')</td><td>12.9122</td></tr><tr><td>(u'real',&nbspu'date')</td><td>12.9122</td></tr><tr><td>(u'la',&nbspu'vega')</td><td>10.8328</td></tr><tr><td>(u'woman',&nbspu'that')</td><td>9.4191</td></tr><tr><td>(u'bookcas',&nbspu'chandler')</td><td>9.4191</td></tr><tr><td>(u'great',&nbspu'butt')</td><td>9.4191</td></tr><tr><td>(u'stair',&nbspu'push')</td><td>9.4191</td></tr><tr><td>(u'ross',&nbspu'grab')</td><td>9.4191</td></tr><tr><td>(u'made',&nbspu'coffe')</td><td>9.4191</td></tr><tr><td>(u'morn',&nbspu'paul')</td><td>9.4191</td></tr></table>"
      ],
      "text/plain": [
       "<ipy_table.IpyTable at 0x103c6a510>"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_table(bigram_tfidf_data[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Season 1, episode 1: (u'cut', u'cut', u'cut'): 65.933, (u'paul', u'wine', u'guy'): 18.838, (u'shoe', u'your', u'shoe'): 12.912, (u'your', u'shoe', u'your'): 12.912, (u'im', u'la', u'vega'):  9.419, (u'push', u'stair', u'push'):  9.419, (u'one', u'woman', u'that'):  9.419, (u'know', u'paul', u'franni'):  9.419, (u'stair', u'push', u'stair'):  9.419, (u'paul', u'yeah', u'joey'):  9.419, (u'ross', u'grab', u'spoon'):  9.419, (u'get', u'go', u'work'):  9.419, (u'ill', u'never', u'grandchildren'):  9.419, (u'hat', u'im', u'say'):  8.608, (u'believ', u'im', u'hear'):  8.608, (u'isnt', u'anyth', u'cant'):  8.608, (u'rachel', u'goodnight', u'ross'):  8.608, (u'back', u'high', u'school'):  7.568, (u'monica', u'yeah', u'yeah'):  6.868, (u'ross', u'got', u'ta'):  5.666\n",
      "\n",
      "Season 1, episode 2: (u'ew', u'ew', u'ew'): 24.044, (u'barri', u'yeah', u'well'):  9.419, (u'well', u'monica', u'ross'):  9.419, (u'gon', u'na', u'helen'):  9.419, (u'god', u'oh', u'god'):  8.959, (u'monica', u'ross', u'your'):  8.608, (u'mr', u'geller', u'oh'):  7.917, (u'oh', u'god', u'oh'):  6.931, (u'mr', u'geller', u'well'):  5.991, (u'that', u'look', u'engag'):  4.710, (u'mindi', u'im', u'happi'):  4.710, (u'yeah', u'think', u'us'):  4.710, (u'ysee', u'he', u'princ'):  4.710, (u'oh', u'martha', u'ludwin'):  4.710, (u'wont', u'chandler', u'joey'):  4.710, (u'ross', u'well', u'wait'):  4.710, (u'dya', u'uh', u'dya'):  4.710, (u'ross', u'mmhmm', u'rachel'):  4.710, (u'marsha', u'well', u'issu'):  4.710, (u'monica', u'know', u'go'):  4.710\n",
      "\n",
      "Season 1, episode 3: (u'thousand', u'dollar', u'footbal'):  9.419, (u'get', u'meet', u'guy'):  9.419, (u'seven', u'thousand', u'dollar'):  9.419, (u'realli', u'like', u'paula'):  9.419, (u'go', u'guy', u'friend'):  9.419, (u'smoke', u'im', u'smoke'):  9.419, (u'im', u'smoke', u'im'):  9.419, (u'dollar', u'footbal', u'phone'):  9.419, (u'dont', u'feel', u'thing'):  9.419, (u'bring', u'guy', u'home'):  9.419, (u'friend', u'realli', u'like'):  9.419, (u'got', u'ta', u'smoke'):  9.419, (u'notnotmin', u'notnotmin', u'notnotmin'):  9.419, (u'weird', u'girl', u'phoeb'):  9.419, (u'guy', u'friend', u'realli'):  9.419, (u'okay', u'okay', u'let'):  7.568, (u'phoeb', u'oh', u'cant'):  6.592, (u'ross', u'hey', u'pheeb'):  5.991, (u'ive', u'got', u'ta'):  5.416, (u'yeah', u'rachel', u'notic'):  4.710\n",
      "\n",
      "Season 1, episode 4: (u'im', u'okay', u'okay'): 17.216, (u'pizza', u'guy', u'yeah'):  9.419, (u'okay', u'got', u'one'):  9.419, (u'see', u'lem', u'see'):  9.419, (u'she', u'walk', u'she'):  9.419, (u'ross', u'peach', u'pit'):  9.419, (u'na', u'know', u'im'):  9.419, (u'visa', u'card', u'peopl'):  9.419, (u'first', u'pay', u'check'):  9.419, (u'card', u'peopl', u'rachel'):  9.419, (u'know', u'im', u'okay'):  9.419, (u'okay', u'okay', u'got'):  8.608, (u'peopl', u'rachel', u'oh'):  8.015, (u'get', u'get', u'get'):  8.015, (u'ross', u'first', u'time'):  7.568, (u'joey', u'ross', u'im'):  7.568, (u'rachel', u'yeah', u'monica'):  4.970, (u'wan', u'na', u'know'):  4.796, (u'duplex', u'rachel', u'what'):  4.710, (u'clinton', u'campaign', u'guy'):  4.710\n",
      "\n",
      "Season 1, episode 5: (u'la', u'la', u'la'): 18.921, (u'hey', u'hey', u'hey'): 14.166, (u'real', u'job', u'go'):  9.419, (u'month', u'call', u'actor'):  9.419, (u'sophist', u'real', u'job'):  9.419, (u'he', u'sophist', u'real'):  9.419, (u'im', u'look', u'im'):  9.419, (u'chandler', u'im', u'fine'):  9.419, (u'woman', u'hey', u'hey'):  9.419, (u'go', u'three', u'audit'):  9.419, (u'job', u'go', u'three'):  9.419, (u'audit', u'month', u'call'):  9.419, (u'look', u'im', u'look'):  9.419, (u'woman', u'yeah', u'well'):  9.419, (u'call', u'actor', u'bob'):  9.419, (u'smart', u'he', u'sophist'):  9.419, (u'night', u'saturday', u'night'):  9.419, (u'three', u'audit', u'month'):  9.419, (u'laundri', u'room', u'build'):  9.419, (u'monica', u'joey', u'monica'):  8.608\n",
      "\n"
     ]
    }
   ],
   "source": [
    "trigram_tfidf_data = calculate_tfidf(trigrammed_data_w_stopwords, count_doc_frequencies(trigrammed_data_w_stopwords))\n",
    "print_tfidf_data(trigram_tfidf_data[:5])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" cellpadding=\"3\" cellspacing=\"0\"  style=\"border:1px solid black;border-collapse:collapse;\"><tr><td>(u'cut',&nbspu'cut',&nbspu'cut')</td><td>65.9334</td></tr><tr><td>(u'paul',&nbspu'wine',&nbspu'guy')</td><td>18.8381</td></tr><tr><td>(u'shoe',&nbspu'your',&nbspu'shoe')</td><td>12.9122</td></tr><tr><td>(u'your',&nbspu'shoe',&nbspu'your')</td><td>12.9122</td></tr><tr><td>(u'im',&nbspu'la',&nbspu'vega')</td><td>9.4191</td></tr><tr><td>(u'push',&nbspu'stair',&nbspu'push')</td><td>9.4191</td></tr><tr><td>(u'one',&nbspu'woman',&nbspu'that')</td><td>9.4191</td></tr><tr><td>(u'know',&nbspu'paul',&nbspu'franni')</td><td>9.4191</td></tr><tr><td>(u'stair',&nbspu'push',&nbspu'stair')</td><td>9.4191</td></tr><tr><td>(u'paul',&nbspu'yeah',&nbspu'joey')</td><td>9.4191</td></tr><tr><td>(u'ross',&nbspu'grab',&nbspu'spoon')</td><td>9.4191</td></tr><tr><td>(u'get',&nbspu'go',&nbspu'work')</td><td>9.4191</td></tr><tr><td>(u'ill',&nbspu'never',&nbspu'grandchildren')</td><td>9.4191</td></tr><tr><td>(u'hat',&nbspu'im',&nbspu'say')</td><td>8.6081</td></tr><tr><td>(u'believ',&nbspu'im',&nbspu'hear')</td><td>8.6081</td></tr><tr><td>(u'isnt',&nbspu'anyth',&nbspu'cant')</td><td>8.6081</td></tr><tr><td>(u'rachel',&nbspu'goodnight',&nbspu'ross')</td><td>8.6081</td></tr><tr><td>(u'back',&nbspu'high',&nbspu'school')</td><td>7.5684</td></tr><tr><td>(u'monica',&nbspu'yeah',&nbspu'yeah')</td><td>6.8680</td></tr><tr><td>(u'ross',&nbspu'got',&nbspu'ta')</td><td>5.6664</td></tr></table>"
      ],
      "text/plain": [
       "<ipy_table.IpyTable at 0x10e1d14d0>"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "make_table(trigram_tfidf_data[0][2])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1,\n",
       " 1,\n",
       " [(u'paul', 117.45961322761602),\n",
       "  (u'franni', 25.824390559225023),\n",
       "  (u'cut', 19.775021196025975),\n",
       "  (u'la', 15.389696144769221),\n",
       "  (u'goodnight', 12.712215321391783),\n",
       "  (u'spoon', 12.364169813433264),\n",
       "  (u'stair', 12.364169813433264),\n",
       "  (u'cmon', 9.729550745276565),\n",
       "  (u'bracketi', 9.419060402624668),\n",
       "  (u'bookcas', 9.419060402624668),\n",
       "  (u'boot', 9.273127360074948),\n",
       "  (u'wine', 8.958797346140274),\n",
       "  (u'push', 8.958797346140274),\n",
       "  (u'vega', 8.671115273688493),\n",
       "  (u'grandchildren', 8.60813018640834),\n",
       "  (u'aura', 8.60813018640834),\n",
       "  (u'shoe', 8.047189562170502),\n",
       "  (u'joann', 8.014666370464942),\n",
       "  (u'flavor', 8.014666370464942),\n",
       "  (u'aruba', 8.014666370464942)])"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tfidf_w_stopwords[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1,\n",
       " 1,\n",
       " [((u'cut', u'cut'), 68.86504149126672),\n",
       "  ((u'wine', u'guy'), 23.54765100656167),\n",
       "  ((u'grab', u'spoon'), 18.838120805249336),\n",
       "  ((u'push', u'stair'), 18.838120805249336),\n",
       "  ((u'paul', u'wine'), 18.838120805249336),\n",
       "  ((u'your', u'shoe'), 17.21626037281668),\n",
       "  ((u'know', u'paul'), 17.21626037281668),\n",
       "  ((u'paul', u'monica'), 17.21626037281668),\n",
       "  ((u'one', u'woman'), 14.443671650576897),\n",
       "  ((u'fifth', u'date'), 14.128590603937003),\n",
       "  ((u'shoe', u'your'), 12.912195279612511),\n",
       "  ((u'real', u'date'), 12.912195279612511),\n",
       "  ((u'la', u'vega'), 10.832753737932673),\n",
       "  ((u'woman', u'that'), 9.419060402624668),\n",
       "  ((u'bookcas', u'chandler'), 9.419060402624668),\n",
       "  ((u'great', u'butt'), 9.419060402624668),\n",
       "  ((u'stair', u'push'), 9.419060402624668),\n",
       "  ((u'ross', u'grab'), 9.419060402624668),\n",
       "  ((u'made', u'coffe'), 9.419060402624668),\n",
       "  ((u'morn', u'paul'), 9.419060402624668)])"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigram_tfidf_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1,\n",
       " 1,\n",
       " [((u'cut', u'cut', u'cut'), 65.93342281837268),\n",
       "  ((u'paul', u'wine', u'guy'), 18.838120805249336),\n",
       "  ((u'shoe', u'your', u'shoe'), 12.912195279612511),\n",
       "  ((u'your', u'shoe', u'your'), 12.912195279612511),\n",
       "  ((u'im', u'la', u'vega'), 9.419060402624668),\n",
       "  ((u'push', u'stair', u'push'), 9.419060402624668),\n",
       "  ((u'one', u'woman', u'that'), 9.419060402624668),\n",
       "  ((u'know', u'paul', u'franni'), 9.419060402624668),\n",
       "  ((u'stair', u'push', u'stair'), 9.419060402624668),\n",
       "  ((u'paul', u'yeah', u'joey'), 9.419060402624668),\n",
       "  ((u'ross', u'grab', u'spoon'), 9.419060402624668),\n",
       "  ((u'get', u'go', u'work'), 9.419060402624668),\n",
       "  ((u'ill', u'never', u'grandchildren'), 9.419060402624668),\n",
       "  ((u'hat', u'im', u'say'), 8.60813018640834),\n",
       "  ((u'believ', u'im', u'hear'), 8.60813018640834),\n",
       "  ((u'isnt', u'anyth', u'cant'), 8.60813018640834),\n",
       "  ((u'rachel', u'goodnight', u'ross'), 8.60813018640834),\n",
       "  ((u'back', u'high', u'school'), 7.568379267836522),\n",
       "  ((u'monica', u'yeah', u'yeah'), 6.8679744089702925),\n",
       "  ((u'ross', u'got', u'ta'), 5.666426688112432)])"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trigram_tfidf_data[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import json\n",
    "with open('ngrams.json', 'w') as f:\n",
    "    to_dump = {\n",
    "        'uni': tfidf_w_stopwords[0][2],\n",
    "        'bi': bigram_tfidf_data[0][2],\n",
    "        'tri': trigram_tfidf_data[0][2]\n",
    "    }\n",
    "    \n",
    "    json.dump(to_dump, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [Root]",
   "language": "python",
   "name": "Python [Root]"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}