Spark Introduction.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MapReduce as a programming model\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fun(x):\n",
    "    return x**2\n",
    "\n",
    "def add(x,y):\n",
    "    return x+y\n",
    "\n",
    "def maximum(x,y):\n",
    "    return max(x,y)\n",
    "\n",
    "def minimum(x,y):\n",
    "    return min(x,y)\n",
    "\n",
    "def prod(x,y):\n",
    "    return x*y\n",
    "\n",
    "def selective_division(x,y):\n",
    "    return x/y if x>y else y/x "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "79\n"
     ]
    }
   ],
   "source": [
    "L = [1,3,4,2,7]\n",
    "\n",
    "s = 0\n",
    "for a in L:\n",
    "    s = s+a**2\n",
    "print(s)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 9, 16, 4, 49]\n"
     ]
    }
   ],
   "source": [
    "    \n",
    "S = map(fun, L)\n",
    "print(S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "selective_division(65, 89)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 9, 16, 4, 49]\n",
      "12\n",
      "79\n",
      "1\n"
     ]
    }
   ],
   "source": [
    "#print(L)\n",
    "print(S)\n",
    "\n",
    "print reduce(selective_division, S)\n",
    "\n",
    "print reduce(add, map(fun, L))\n",
    "\n",
    "\n",
    "print reduce(minimum, S)\n",
    "#print reduce(maximum, S)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Number of even entries "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def is_even(x):\n",
    "    return 1 if x%2==0 else 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "2"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "reduce(add, map(is_even, L))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'AhmetMehmetveli'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "A = ['Ahmet', 'Mehmet', 'veli'] \n",
    "\n",
    "def is_A(x):\n",
    "    return 1 if x[0]=='A' else 0\n",
    "\n",
    "reduce(lambda x,y: x+y, map(is_A, A))\n",
    "\n",
    "reduce(add, A)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Spark\n",
    "\n",
    "RDD : Resilient Distributed Dataset\n",
    "\n",
    "Note: This notebook version needs the Python 2.7. When the notebook server is launched from\n",
    "a 3.x environment, the environment variables are not set correctly."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Load a text file on the HDFS as a RDD\n",
    "* Display number of lines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "ename": "Py4JJavaError",
     "evalue": "An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost, executor driver): org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/lib/pyspark.zip/pyspark/worker.py\", line 122, in main\n    (\"%d.%d\" % sys.version_info[:2], version))\nException: Python in worker has different version 3.6 than that in driver 2.7, PySpark cannot run with different minor versions.Please check environment variables PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON are correctly set.\n\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRDD.scala:193)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.<init>(PythonRDD.scala:234)\n\tat org.apache.spark.api.python.PythonRunner.compute(PythonRDD.scala:152)\n\tat org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:63)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:287)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1928)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1941)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1954)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1968)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:362)\n\tat org.apache.spark.rdd.RDD.collect(RDD.scala:935)\n\tat org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:453)\n\tat org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:483)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:280)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:214)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/lib/pyspark.zip/pyspark/worker.py\", line 122, in main\n    (\"%d.%d\" % sys.version_info[:2], version))\nException: Python in worker has different version 3.6 than that in driver 2.7, PySpark cannot run with different minor versions.Please check environment variables PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON are correctly set.\n\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRDD.scala:193)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.<init>(PythonRDD.scala:234)\n\tat org.apache.spark.api.python.PythonRunner.compute(PythonRDD.scala:152)\n\tat org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:63)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:287)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\t... 1 more\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mPy4JJavaError\u001b[0m                             Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-6-2ccd5a148474>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0mtextFile\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0msc\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtextFile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"data/books-eng/hamlet.txt\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      5\u001b[0m \u001b[0;31m#textFile = sc.textFile(\"data/books-eng\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 6\u001b[0;31m \u001b[0mtextFile\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/pyspark/rdd.pyc\u001b[0m in \u001b[0;36mcount\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1038\u001b[0m         \u001b[0;36m3\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1039\u001b[0m         \"\"\"\n\u001b[0;32m-> 1040\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmapPartitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0m_\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mi\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1041\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1042\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mstats\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/pyspark/rdd.pyc\u001b[0m in \u001b[0;36msum\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m   1029\u001b[0m         \u001b[0;36m6.0\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1030\u001b[0m         \"\"\"\n\u001b[0;32m-> 1031\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmapPartitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;32mlambda\u001b[0m \u001b[0mx\u001b[0m\u001b[0;34m:\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0msum\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mfold\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0moperator\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0madd\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1032\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1033\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mcount\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/pyspark/rdd.pyc\u001b[0m in \u001b[0;36mfold\u001b[0;34m(self, zeroValue, op)\u001b[0m\n\u001b[1;32m    903\u001b[0m         \u001b[0;31m# zeroValue provided to each partition is unique from the one provided\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    904\u001b[0m         \u001b[0;31m# to the final reduce call\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 905\u001b[0;31m         \u001b[0mvals\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mmapPartitions\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollect\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    906\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mreduce\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mop\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvals\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mzeroValue\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    907\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/pyspark/rdd.pyc\u001b[0m in \u001b[0;36mcollect\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m    806\u001b[0m         \"\"\"\n\u001b[1;32m    807\u001b[0m         \u001b[0;32mwith\u001b[0m \u001b[0mSCCallSiteSync\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcontext\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mcss\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 808\u001b[0;31m             \u001b[0mport\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mctx\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jvm\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPythonRDD\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcollectAndServe\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jrdd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mrdd\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    809\u001b[0m         \u001b[0;32mreturn\u001b[0m \u001b[0mlist\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0m_load_from_socket\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mport\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_jrdd_deserializer\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip/py4j/java_gateway.py\u001b[0m in \u001b[0;36m__call__\u001b[0;34m(self, *args)\u001b[0m\n\u001b[1;32m   1131\u001b[0m         \u001b[0manswer\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mgateway_client\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0msend_command\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1132\u001b[0m         return_value = get_return_value(\n\u001b[0;32m-> 1133\u001b[0;31m             answer, self.gateway_client, self.target_id, self.name)\n\u001b[0m\u001b[1;32m   1134\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1135\u001b[0m         \u001b[0;32mfor\u001b[0m \u001b[0mtemp_arg\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mtemp_args\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/pyspark/sql/utils.pyc\u001b[0m in \u001b[0;36mdeco\u001b[0;34m(*a, **kw)\u001b[0m\n\u001b[1;32m     61\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mdeco\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     62\u001b[0m         \u001b[0;32mtry\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 63\u001b[0;31m             \u001b[0;32mreturn\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m*\u001b[0m\u001b[0ma\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkw\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     64\u001b[0m         \u001b[0;32mexcept\u001b[0m \u001b[0mpy4j\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mprotocol\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mPy4JJavaError\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     65\u001b[0m             \u001b[0ms\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0me\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mjava_exception\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mtoString\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/lib/py4j-0.10.4-src.zip/py4j/protocol.py\u001b[0m in \u001b[0;36mget_return_value\u001b[0;34m(answer, gateway_client, target_id, name)\u001b[0m\n\u001b[1;32m    317\u001b[0m                 raise Py4JJavaError(\n\u001b[1;32m    318\u001b[0m                     \u001b[0;34m\"An error occurred while calling {0}{1}{2}.\\n\"\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 319\u001b[0;31m                     format(target_id, \".\", name), value)\n\u001b[0m\u001b[1;32m    320\u001b[0m             \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    321\u001b[0m                 raise Py4JError(\n",
      "\u001b[0;31mPy4JJavaError\u001b[0m: An error occurred while calling z:org.apache.spark.api.python.PythonRDD.collectAndServe.\n: org.apache.spark.SparkException: Job aborted due to stage failure: Task 0 in stage 0.0 failed 1 times, most recent failure: Lost task 0.0 in stage 0.0 (TID 0, localhost, executor driver): org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/lib/pyspark.zip/pyspark/worker.py\", line 122, in main\n    (\"%d.%d\" % sys.version_info[:2], version))\nException: Python in worker has different version 3.6 than that in driver 2.7, PySpark cannot run with different minor versions.Please check environment variables PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON are correctly set.\n\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRDD.scala:193)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.<init>(PythonRDD.scala:234)\n\tat org.apache.spark.api.python.PythonRunner.compute(PythonRDD.scala:152)\n\tat org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:63)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:287)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\tat java.lang.Thread.run(Thread.java:745)\n\nDriver stacktrace:\n\tat org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1435)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1423)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1422)\n\tat scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59)\n\tat scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1422)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)\n\tat org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:802)\n\tat scala.Option.foreach(Option.scala:257)\n\tat org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:802)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1650)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1605)\n\tat org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1594)\n\tat org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48)\n\tat org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:628)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1928)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1941)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1954)\n\tat org.apache.spark.SparkContext.runJob(SparkContext.scala:1968)\n\tat org.apache.spark.rdd.RDD$$anonfun$collect$1.apply(RDD.scala:936)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)\n\tat org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)\n\tat org.apache.spark.rdd.RDD.withScope(RDD.scala:362)\n\tat org.apache.spark.rdd.RDD.collect(RDD.scala:935)\n\tat org.apache.spark.api.python.PythonRDD$.collectAndServe(PythonRDD.scala:453)\n\tat org.apache.spark.api.python.PythonRDD.collectAndServe(PythonRDD.scala)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)\n\tat sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)\n\tat sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)\n\tat java.lang.reflect.Method.invoke(Method.java:483)\n\tat py4j.reflection.MethodInvoker.invoke(MethodInvoker.java:244)\n\tat py4j.reflection.ReflectionEngine.invoke(ReflectionEngine.java:357)\n\tat py4j.Gateway.invoke(Gateway.java:280)\n\tat py4j.commands.AbstractCommand.invokeMethod(AbstractCommand.java:132)\n\tat py4j.commands.CallCommand.execute(CallCommand.java:79)\n\tat py4j.GatewayConnection.run(GatewayConnection.java:214)\n\tat java.lang.Thread.run(Thread.java:745)\nCaused by: org.apache.spark.api.python.PythonException: Traceback (most recent call last):\n  File \"/Users/cemgil/spark-2.0.1-bin-hadoop2.7/python/lib/pyspark.zip/pyspark/worker.py\", line 122, in main\n    (\"%d.%d\" % sys.version_info[:2], version))\nException: Python in worker has different version 3.6 than that in driver 2.7, PySpark cannot run with different minor versions.Please check environment variables PYSPARK_PYTHON and PYSPARK_DRIVER_PYTHON are correctly set.\n\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.read(PythonRDD.scala:193)\n\tat org.apache.spark.api.python.PythonRunner$$anon$1.<init>(PythonRDD.scala:234)\n\tat org.apache.spark.api.python.PythonRunner.compute(PythonRDD.scala:152)\n\tat org.apache.spark.api.python.PythonRDD.compute(PythonRDD.scala:63)\n\tat org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:323)\n\tat org.apache.spark.rdd.RDD.iterator(RDD.scala:287)\n\tat org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87)\n\tat org.apache.spark.scheduler.Task.run(Task.scala:99)\n\tat org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:325)\n\tat java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)\n\tat java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)\n\t... 1 more\n"
     ]
    }
   ],
   "source": [
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "textFile = sc.textFile(\"data/books-eng/hamlet.txt\")\n",
    "#textFile = sc.textFile(\"data/books-eng\")\n",
    "textFile.count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get the first line"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'Hamlet, Prince of Denmark '"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.first()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find number of lines where a word appears"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "121"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "word = \"CLAUDIUS\"\n",
    "textFile.filter(lambda lin: word in lin).count()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Get a random sample from the file"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "u'A Priest. '"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.sample(withReplacement=False, fraction=0.05).first()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[u'',\n",
       " u'HORATIO friend to Hamlet. ',\n",
       " u'',\n",
       " u'OPHELIA daughter to Polonius. ',\n",
       " u'',\n",
       " u'   And liegemen to the Dane.',\n",
       " u'',\n",
       " u'',\n",
       " u'',\n",
       " u'']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def prnt(x):\n",
    "    print x\n",
    "\n",
    "textFile.sample(withReplacement=False, fraction=0.05).take(10)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find the line with most words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "122"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.map(lambda line: len(line.split())).reduce(lambda a, b: a if (a > b) else b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(122,\n",
       " u\"    [Enter a King and a Queen very lovingly; the Queen embracing him, and he her. She kneels, and makes show of protestation unto him. He takes her up, and declines his head upon her neck: lays him down upon a bank of flowers: she, seeing him asleep, leaves him. Anon comes in a fellow, takes off his crown, kisses it, and pours poison in the King's ears, and exit. The Queen returns; finds the King dead, and makes passionate action. The Poisoner, with some two or three Mutes, comes in again, seeming to lament with her. The dead body is carried away. The Poisoner wooes the Queen with gifts: she seems loath and unwilling awhile, but in the end accepts his love]\")"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.map(lambda line: (len(line.split()),line)).reduce(lambda a, b: a if (a[0] > b[0]) else b)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['abc', 'df']"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'abc df'.split()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Counting words"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'stays:', 1),\n",
       " (u'pardon', 4),\n",
       " (u'admiration?', 1),\n",
       " (u'all', 88),\n",
       " (u'foul', 9),\n",
       " (u'four', 4),\n",
       " (u'sending.', 1),\n",
       " (u'appetite', 1),\n",
       " (u'hate', 1),\n",
       " (u'Francisco.', 1),\n",
       " (u'up.', 3),\n",
       " (u'time,', 8),\n",
       " (u'grace,', 4),\n",
       " (u\"o'er-raught\", 1),\n",
       " (u'Blasting', 1),\n",
       " (u'time.', 4),\n",
       " (u'common.', 1),\n",
       " (u'up:', 1),\n",
       " (u'whatsoever', 1),\n",
       " (u'fifty,', 1),\n",
       " (u'sorry', 2),\n",
       " (u'[Leaps', 2),\n",
       " (u'pound.', 1),\n",
       " (u'flushing', 1),\n",
       " (u'drunkards,', 1),\n",
       " (u'quarrelling,', 1),\n",
       " (u\"thou'rt\", 1),\n",
       " (u'Hamlet,', 25),\n",
       " (u'Hamlet.', 11),\n",
       " (u'every', 10),\n",
       " (u'bonnet', 1),\n",
       " (u'advice;', 1),\n",
       " (u'Old', 1),\n",
       " (u'fingers,', 1),\n",
       " (u'bringing', 1),\n",
       " (u'encounter', 1),\n",
       " (u'jointress', 1),\n",
       " (u\"Hamlet.'\", 1),\n",
       " (u'Bernardo,', 1),\n",
       " (u'rages,', 1),\n",
       " (u'offend', 1),\n",
       " (u'companies', 1),\n",
       " (u'wards', 1),\n",
       " (u'ending,', 1),\n",
       " (u'abatements', 1),\n",
       " (u'convenience', 1),\n",
       " (u'ungored.', 1),\n",
       " (u'Go', 12),\n",
       " (u'sleep', 3),\n",
       " (u'Beware', 1),\n",
       " (u'triumph', 1),\n",
       " (u'There,', 2),\n",
       " (u'Gracious,', 1),\n",
       " (u'force', 2),\n",
       " (u\"unsinew'd,\", 1),\n",
       " (u'feathers', 1),\n",
       " (u'murdering-piece,', 1),\n",
       " (u'Unite', 1),\n",
       " (u'her;', 1),\n",
       " (u'hundred.', 1),\n",
       " (u'her?', 3),\n",
       " (u'valiant', 2),\n",
       " (u'others,', 1),\n",
       " (u'tether', 1),\n",
       " (u'me;', 6),\n",
       " (u'even', 14),\n",
       " (u'aim', 1),\n",
       " (u'maggots:', 1),\n",
       " (u'hide', 3),\n",
       " (u'leave?', 2),\n",
       " (u'solemn', 3),\n",
       " (u'general:', 1),\n",
       " (u'near', 6),\n",
       " (u'sulphurous', 1),\n",
       " (u'favours.', 1),\n",
       " (u'This', 45),\n",
       " (u'change;', 1),\n",
       " (u'pickers', 1),\n",
       " (u'above', 2),\n",
       " (u'majesties', 2),\n",
       " (u'rest!', 2),\n",
       " (u'ever', 11),\n",
       " (u'nature:', 1),\n",
       " (u'insinuation', 1),\n",
       " (u'behavior', 1),\n",
       " (u'Conceit', 2),\n",
       " (u'Drabbing:', 1),\n",
       " (u'up,', 3),\n",
       " (u'here', 27),\n",
       " (u'acted;', 1),\n",
       " (u'stopping', 1),\n",
       " (u'mew', 1),\n",
       " (u'pate', 4),\n",
       " (u'gaffs', 1),\n",
       " (u'wedding.', 1),\n",
       " (u'Addicted', 1),\n",
       " (u'hat]', 1),\n",
       " (u'battery?', 1),\n",
       " (u'mistress!', 1),\n",
       " (u'beloved;', 1),\n",
       " (u'daughter', 9),\n",
       " (u'fay,', 1),\n",
       " (u'study', 1),\n",
       " (u'light.', 1),\n",
       " (u'Hail', 1),\n",
       " (u\"seal'd?\", 1),\n",
       " (u'Hillo,', 2),\n",
       " (u'chalice', 1),\n",
       " (u'light:', 1),\n",
       " (u'prison.', 2),\n",
       " (u'settled', 1),\n",
       " (u'battalions.', 1),\n",
       " (u'widow,', 1),\n",
       " (u'circumstance,', 1),\n",
       " (u'sledded', 1),\n",
       " (u'heedful', 1),\n",
       " (u'imputation', 1),\n",
       " (u'Giving', 2),\n",
       " (u'brought', 5),\n",
       " (u'throat', 2),\n",
       " (u'Behind', 2),\n",
       " (u'moral', 1),\n",
       " (u'god-like', 1),\n",
       " (u'assay!', 1),\n",
       " (u'Whips', 1),\n",
       " (u'gentleman?', 2),\n",
       " (u'reply.', 1),\n",
       " (u'fire;', 1),\n",
       " (u'gentleman;', 1),\n",
       " (u'self-slaughter!', 1),\n",
       " (u\"soil'd\", 1),\n",
       " (u'goose-quills', 1),\n",
       " (u'would', 64),\n",
       " (u'remainder', 1),\n",
       " (u'aloof;', 1),\n",
       " (u'exhort', 1),\n",
       " (u'tenders', 2),\n",
       " (u'his:', 1),\n",
       " (u'call', 20),\n",
       " (u\"have't.\", 1),\n",
       " (u'strike', 1),\n",
       " (u'right.', 2),\n",
       " (u'springe,', 1),\n",
       " (u\"to't\", 2),\n",
       " (u'calf', 1),\n",
       " (u'GHOST', 1),\n",
       " (u'thrift,', 2),\n",
       " (u'duty?', 1),\n",
       " (u'lungs', 1),\n",
       " (u'engaged!', 1),\n",
       " (u'off;', 1),\n",
       " (u'wings,', 1),\n",
       " (u'whirling', 1),\n",
       " (u'canonized', 1),\n",
       " (u'grief', 8),\n",
       " (u'[Elsinore.', 2),\n",
       " (u'sting', 2),\n",
       " (u'continual', 1),\n",
       " (u'Feeds', 1),\n",
       " (u'Ophelia,', 8),\n",
       " (u'unpack', 1),\n",
       " (u'mirth,', 1),\n",
       " (u'me', 141),\n",
       " (u'confound', 1),\n",
       " (u'join', 2),\n",
       " (u'Guildenstern!', 3),\n",
       " (u'pursue', 2),\n",
       " (u'ratifiers', 1),\n",
       " (u'sight?', 1),\n",
       " (u'unproportioned', 1),\n",
       " (u'roof', 1),\n",
       " (u'majesty;', 1),\n",
       " (u'greenly,', 1),\n",
       " (u'arm,', 1),\n",
       " (u'foils', 2),\n",
       " (u'liar;', 1),\n",
       " (u'brows;', 1),\n",
       " (u'Guildenstern?', 1),\n",
       " (u'spendthrift', 1),\n",
       " (u'Guildenstern;', 1),\n",
       " (u'favour,', 1),\n",
       " (u'la,', 1),\n",
       " (u'bevy', 1),\n",
       " (u'exercises;', 1),\n",
       " (u\"beggar'd,\", 1),\n",
       " (u'serpent', 2),\n",
       " (u'loosed', 1),\n",
       " (u'Charity,', 1),\n",
       " (u'dumb;', 1),\n",
       " (u'honey', 1),\n",
       " (u'emphasis?', 1),\n",
       " (u'follows,', 2),\n",
       " (u'Holding', 1),\n",
       " (u\"by'r\", 1),\n",
       " (u'appointment', 1),\n",
       " (u'blank,', 1),\n",
       " (u'cautel', 1),\n",
       " (u'ceremony', 2),\n",
       " (u'rites?', 1),\n",
       " (u'end', 10),\n",
       " (u'Hamlet:', 5),\n",
       " (u'keep,', 2),\n",
       " (u'said,', 3),\n",
       " (u'travel', 1),\n",
       " (u\"honour'd\", 3),\n",
       " (u'rouse,', 1),\n",
       " (u'very', 57),\n",
       " (u'hot', 5),\n",
       " (u'win.', 1),\n",
       " (u'riotous', 1),\n",
       " (u'answer', 8),\n",
       " (u'shortens', 1),\n",
       " (u'A', 66),\n",
       " (u'since,', 2),\n",
       " (u'beauty', 4),\n",
       " (u'Wilt', 1),\n",
       " (u'Eyes', 1),\n",
       " (u'Will', 17),\n",
       " (u'follows:', 1),\n",
       " (u'forty,', 1),\n",
       " (u'fat', 5),\n",
       " (u'foh!', 1),\n",
       " (u'conclusions,', 1),\n",
       " (u'contrive', 2),\n",
       " (u'muddy', 1),\n",
       " (u'[Makes', 1),\n",
       " (u'worm.', 1),\n",
       " (u'All', 12),\n",
       " (u'cold;', 1),\n",
       " (u'third', 2),\n",
       " (u'hasty', 1),\n",
       " (u'post-haste', 1),\n",
       " (u\"embark'd:\", 1),\n",
       " (u'otherwise:', 1),\n",
       " (u'greet', 1),\n",
       " (u'dire', 1),\n",
       " (u'Ilium,', 1),\n",
       " (u'Art', 1),\n",
       " (u'still.', 1),\n",
       " (u'haste', 8),\n",
       " (u'Hyrcanian', 1),\n",
       " (u'fault:', 1),\n",
       " (u'sun.', 1),\n",
       " (u'centre.', 1),\n",
       " (u'wind', 9),\n",
       " (u'thirty', 3),\n",
       " (u'origin', 2),\n",
       " (u'modesty', 3),\n",
       " (u'Hide', 1),\n",
       " (u\"find'st\", 1),\n",
       " (u'office', 1),\n",
       " (u'over', 2),\n",
       " (u'unwrung.', 1),\n",
       " (u'throne;', 1),\n",
       " (u'drab,', 1),\n",
       " (u'ears,', 2),\n",
       " (u\"but'\", 1),\n",
       " (u'ears.', 1),\n",
       " (u'bound,', 2),\n",
       " (u'bubbles', 1),\n",
       " (u'clapped', 1),\n",
       " (u'forehead', 2),\n",
       " (u'disease,', 1),\n",
       " (u'then.', 1),\n",
       " (u'then,', 15),\n",
       " (u'Attendants]', 6),\n",
       " (u\"What's\", 6),\n",
       " (u'Here', 6),\n",
       " (u'better', 12),\n",
       " (u'Discomfort', 1),\n",
       " (u'vast', 1),\n",
       " (u'feed?', 1),\n",
       " (u'whips', 1),\n",
       " (u\"baker's\", 1),\n",
       " (u'overcome', 1),\n",
       " (u'welcome,', 3),\n",
       " (u'Lest', 1),\n",
       " (u'them', 43),\n",
       " (u'bawdry,', 1),\n",
       " (u'combination', 1),\n",
       " (u'regard,', 1),\n",
       " (u'thee', 34),\n",
       " (u'history,', 1),\n",
       " (u\"we'll\", 5),\n",
       " (u'history.', 1),\n",
       " (u'break', 3),\n",
       " (u'Anon', 1),\n",
       " (u'short:', 1),\n",
       " (u'they', 78),\n",
       " (u'frock', 1),\n",
       " (u'thyself;', 1),\n",
       " (u'advise', 1),\n",
       " (u'bank', 1),\n",
       " (u'England;', 4),\n",
       " (u'mercy:', 1),\n",
       " (u\"better'd,\", 1),\n",
       " (u'gallows', 3),\n",
       " (u'forgiveness', 1),\n",
       " (u'beer-barrel?', 1),\n",
       " (u'persever', 1),\n",
       " (u'REYNALDO', 14),\n",
       " (u'herald', 1),\n",
       " (u'--', 136),\n",
       " (u'week;', 1),\n",
       " (u'proceeding', 1),\n",
       " (u'arrow', 1),\n",
       " (u'Black', 1),\n",
       " (u'went', 5),\n",
       " (u'hems,', 1),\n",
       " (u'expectancy', 1),\n",
       " (u'lifted', 1),\n",
       " (u'Perchance', 1),\n",
       " (u'oped', 1),\n",
       " (u\"'ild\", 1),\n",
       " (u'laying', 2),\n",
       " (u'Confess', 1),\n",
       " (u'ministering', 1),\n",
       " (u'short,', 1),\n",
       " (u'Gentleman]', 2),\n",
       " (u\"Pull'd\", 1),\n",
       " (u'truant', 1),\n",
       " (u'array', 1),\n",
       " (u'is;', 3),\n",
       " (u'day:', 1),\n",
       " (u'potion.', 1),\n",
       " (u'list!', 1),\n",
       " (u\"'em?\", 2),\n",
       " (u'foils;', 1),\n",
       " (u'content', 2),\n",
       " (u'day,', 5),\n",
       " (u'hobby-horse,', 1),\n",
       " (u'unyoke.', 1),\n",
       " (u'heavenly', 2),\n",
       " (u'perturbed', 1),\n",
       " (u'how.', 1),\n",
       " (u'forth', 2),\n",
       " (u'Fare', 4),\n",
       " (u'bands.', 1),\n",
       " (u'churches,', 1),\n",
       " (u'assured', 1),\n",
       " (u'ghost!', 1),\n",
       " (u'Alas,', 9),\n",
       " (u'lawless', 2),\n",
       " (u'given', 9),\n",
       " (u'free', 7),\n",
       " (u'aid:', 1),\n",
       " (u'bestow', 3),\n",
       " (u'thyself?', 1),\n",
       " (u'rain', 1),\n",
       " (u'born,', 1),\n",
       " (u'be.', 5),\n",
       " (u'mistress', 2),\n",
       " (u'Soft', 1),\n",
       " (u\"if't\", 2),\n",
       " (u'rebels,', 1),\n",
       " (u'jot', 1),\n",
       " (u'strewments', 1),\n",
       " (u'fellow:', 1),\n",
       " (u'rewards,', 1),\n",
       " (u'Rome,', 2),\n",
       " (u'mines,', 1),\n",
       " (u'fellow,', 1),\n",
       " (u'bears', 1),\n",
       " (u'shipped', 1),\n",
       " (u'prodigal', 2),\n",
       " (u'Dost', 5),\n",
       " (u'go;', 2),\n",
       " (u'moving', 1),\n",
       " (u'loud', 1),\n",
       " (u'baby;', 1),\n",
       " (u'within,', 1),\n",
       " (u'grief;', 2),\n",
       " (u'Alexander,', 1),\n",
       " (u'render', 1),\n",
       " (u'too.', 5),\n",
       " (u'accent', 2),\n",
       " (u'too,', 4),\n",
       " (u'rank', 6),\n",
       " (u'congregation', 1),\n",
       " (u'yet,', 4),\n",
       " (u'another', 6),\n",
       " (u'If,', 1),\n",
       " (u'me!', 4),\n",
       " (u'thick', 1),\n",
       " (u'too:', 1),\n",
       " (u'Welcome,', 4),\n",
       " (u'answer;', 1),\n",
       " (u'acquaint', 1),\n",
       " (u'Sirs,', 1),\n",
       " (u'easy;', 1),\n",
       " (u'can:', 1),\n",
       " (u'top', 6),\n",
       " (u'shipwrights,', 1),\n",
       " (u'nation.', 1),\n",
       " (u'power;', 1),\n",
       " (u\"i'\", 23),\n",
       " (u'blossoms', 1),\n",
       " (u'bitter', 2),\n",
       " (u'pooh!', 1),\n",
       " (u'naught:', 1),\n",
       " (u'spilt.', 1),\n",
       " (u'griefs', 1),\n",
       " (u'serve', 2),\n",
       " (u'satyr;', 1),\n",
       " (u'scant', 1),\n",
       " (u'somewhat', 1),\n",
       " (u'kept', 4),\n",
       " (u'feature,', 1),\n",
       " (u'brains,', 1),\n",
       " (u'fiction,', 1),\n",
       " (u'Italian:', 1),\n",
       " (u'positively', 1),\n",
       " (u'begins', 2),\n",
       " (u'dog,', 1),\n",
       " (u'affrighted!', 1),\n",
       " (u'Wittenberg,', 2),\n",
       " (u'dungeons,', 1),\n",
       " (u'slain?', 1),\n",
       " (u'Termagant;', 1),\n",
       " (u'air,', 5),\n",
       " (u'second', 9),\n",
       " (u'air.', 3),\n",
       " (u'height,', 1),\n",
       " (u'awhile;', 3),\n",
       " (u'See', 2),\n",
       " (u'to.', 3),\n",
       " (u\"'Adieu,\", 1),\n",
       " (u'to,', 3),\n",
       " (u'combat;', 1),\n",
       " (u'well?', 2),\n",
       " (u'wanton,', 1),\n",
       " (u'sighs,', 1),\n",
       " (u'well;', 4),\n",
       " (u'room.', 1),\n",
       " (u\"mind's\", 2),\n",
       " (u'[Exit]', 19),\n",
       " (u'mind', 7),\n",
       " (u'mining', 1),\n",
       " (u'yaw', 1),\n",
       " (u'door]', 1),\n",
       " (u'here!', 2),\n",
       " (u'controversy:', 1),\n",
       " (u'Angels', 1),\n",
       " (u'me?', 9),\n",
       " (u'wagered', 1),\n",
       " (u'Confound', 1),\n",
       " (u'mightier:', 1),\n",
       " (u'wholesome', 6),\n",
       " (u'on,', 6),\n",
       " (u'do,', 13),\n",
       " (u'What?', 1),\n",
       " (u'dove!', 1),\n",
       " (u'cheer', 3),\n",
       " (u'impotent', 1),\n",
       " (u\"Let's\", 4),\n",
       " (u'fitting', 1),\n",
       " (u'instructs', 1),\n",
       " (u'They', 17),\n",
       " (u'willing,', 2),\n",
       " (u'grow:', 1),\n",
       " (u'point:', 1),\n",
       " (u'tell.', 2),\n",
       " (u'rest;', 1),\n",
       " (u'tell,', 1),\n",
       " (u'long?', 2),\n",
       " (u'withdraw', 1),\n",
       " (u'maiden', 2),\n",
       " (u'see?', 1),\n",
       " (u'much,', 3),\n",
       " (u'accurst!', 1),\n",
       " (u'what', 112),\n",
       " (u'glimpses', 1),\n",
       " (u'Denmark,', 3),\n",
       " (u'see!', 1),\n",
       " (u'following;', 1),\n",
       " (u'him:', 9),\n",
       " (u'roar?', 1),\n",
       " (u'Comes', 1),\n",
       " (u\"grows,'\", 1),\n",
       " (u'records,', 1),\n",
       " (u'unripe,', 1),\n",
       " (u'affair:', 1),\n",
       " (u'knows,', 2),\n",
       " (u'metal', 1),\n",
       " (u'breathes', 1),\n",
       " (u\"return'd,\", 1),\n",
       " (u'nature,', 8),\n",
       " (u\"return'd.\", 1),\n",
       " (u\"me'?\", 1),\n",
       " (u'subject:', 1),\n",
       " (u'with;', 1),\n",
       " (u'affair.', 1),\n",
       " (u'came', 8),\n",
       " (u'a-down', 1),\n",
       " (u'cunnings:', 1),\n",
       " (u'celebrated,', 1),\n",
       " (u'Unsifted', 1),\n",
       " (u'liest', 2),\n",
       " (u'Denmark:', 2),\n",
       " (u'fennel', 1),\n",
       " (u'cannot,', 1),\n",
       " (u'Devoutly', 1),\n",
       " (u'cannot.', 3),\n",
       " (u'wrung', 1),\n",
       " (u'gore,', 1),\n",
       " (u'mellow', 1),\n",
       " (u'Impart.', 1),\n",
       " (u'ear,', 4),\n",
       " (u'ear.', 3),\n",
       " (u'divinity', 2),\n",
       " (u'just', 2),\n",
       " (u'advanced', 1),\n",
       " (u'rabble', 1),\n",
       " (u'Bid', 1),\n",
       " (u'Free', 1),\n",
       " (u'commandment', 3),\n",
       " (u'theme', 2),\n",
       " (u'hither?', 2),\n",
       " (u'afar', 1),\n",
       " (u'ear:', 1),\n",
       " (u'heartily', 1),\n",
       " (u'worser', 1),\n",
       " (u'funeral', 3),\n",
       " (u\"dispatch'd:\", 1),\n",
       " (u'steel', 1),\n",
       " (u'spheres,', 1),\n",
       " (u'mouths', 1),\n",
       " (u'knowest', 1),\n",
       " (u'stop', 3),\n",
       " (u'lecherous,', 1),\n",
       " (u'leisure', 1),\n",
       " (u'command?', 1),\n",
       " (u'Leave', 3),\n",
       " (u'Put', 2),\n",
       " (u\"watch'd.\", 1),\n",
       " (u'report', 5),\n",
       " (u'glow-worm', 1),\n",
       " (u'nasty', 1),\n",
       " (u'comply', 2),\n",
       " (u'sorrows', 1),\n",
       " (u'runs', 2),\n",
       " (u\"unmatch'd\", 1),\n",
       " (u'chase.', 1),\n",
       " (u'Stay,', 1),\n",
       " (u'Wherein,', 1),\n",
       " (u'Officers,', 1),\n",
       " (u'them;', 4),\n",
       " (u'men,', 3),\n",
       " (u'men.', 1),\n",
       " (u'seeks', 1),\n",
       " (u'cousin,', 1),\n",
       " (u'warning,', 1),\n",
       " (u'ban', 1),\n",
       " (u\"be:'\", 1),\n",
       " (u'steal', 1),\n",
       " (u'brothers.', 1),\n",
       " (u\"o'er,\", 1),\n",
       " (u'healthful', 1),\n",
       " (u'unworthy', 2),\n",
       " (u'fair', 16),\n",
       " (u'churlish', 1),\n",
       " (u'nut', 1),\n",
       " (u'offended.', 3),\n",
       " (u'observed', 1),\n",
       " (u'Taken', 1),\n",
       " (u'depends', 1),\n",
       " (u'winks,', 1),\n",
       " (u'gentry,', 1),\n",
       " (u'fain', 4),\n",
       " (u'best', 9),\n",
       " (u'wits,', 1),\n",
       " (u'wits.', 1),\n",
       " (u'draws', 2),\n",
       " (u'God.', 1),\n",
       " (u'face,', 2),\n",
       " (u'away', 8),\n",
       " (u'lain', 1),\n",
       " (u'gentleman', 5),\n",
       " (u'unsmirched', 1),\n",
       " (u'rat?', 1),\n",
       " (u'skull]', 4),\n",
       " (u'capons', 1),\n",
       " (u'scorn', 1),\n",
       " (u'conceit?', 1),\n",
       " (u'in,', 3),\n",
       " (u'foe,', 1),\n",
       " (u'hence', 3),\n",
       " (u'fit;', 1),\n",
       " (u'accord', 1),\n",
       " (u'terms', 3),\n",
       " (u'loves', 3),\n",
       " (u'mincing', 1),\n",
       " (u'torrent,', 1),\n",
       " (u'weak', 2),\n",
       " (u'jaws,', 1),\n",
       " (u\"flatter'd?\", 1),\n",
       " (u'extent', 1),\n",
       " (u'Farewell!', 1),\n",
       " (u'pity', 2),\n",
       " (u'essentially', 1),\n",
       " (u'translate:', 1),\n",
       " (u'malefactions;', 1),\n",
       " (u'fire!', 1),\n",
       " (u'steep', 1),\n",
       " (u'bray', 1),\n",
       " (u'pressure.', 1),\n",
       " (u'against', 18),\n",
       " (u'pronounced:', 1),\n",
       " (u'encounter;', 1),\n",
       " (u'bet', 1),\n",
       " (u\"seal'd\", 4),\n",
       " (u'foot;', 1),\n",
       " (u'nay,', 3),\n",
       " (u'and', 697),\n",
       " (u'Given', 1),\n",
       " (u\"labour'd\", 1),\n",
       " (u'doubt,', 1),\n",
       " (u'royal', 3),\n",
       " (u'vain', 1),\n",
       " (u'queen.', 2),\n",
       " (u'enterprise,', 1),\n",
       " (u'directions', 1),\n",
       " (u'non', 1),\n",
       " (u\"match'd,\", 1),\n",
       " (u'Mother,', 4),\n",
       " (u'now,', 23),\n",
       " (u'now.', 3),\n",
       " (u'murder,', 2),\n",
       " (u'fall:', 1),\n",
       " (u'puts', 4),\n",
       " (u'Pyrrhus', 4),\n",
       " (u'union', 2),\n",
       " (u'buffets', 1),\n",
       " (u'been', 25),\n",
       " (u'Subscribed', 1),\n",
       " (u'murder:', 1),\n",
       " (u'gambol', 1),\n",
       " (u'interpret', 1),\n",
       " (u'vows:', 1),\n",
       " (u'nomination', 1),\n",
       " (u'[Behind]', 2),\n",
       " (u'appears:', 1),\n",
       " (u'duties', 1),\n",
       " (u\"'Anon\", 1),\n",
       " (u'dry', 3),\n",
       " (u\"follow'd\", 2),\n",
       " (u'easy', 1),\n",
       " (u'spring,', 1),\n",
       " (u'drugs', 1),\n",
       " (u'Folded', 1),\n",
       " (u'speeches', 1),\n",
       " (u'church:', 1),\n",
       " (u\"Revisit'st\", 1),\n",
       " (u'catch', 2),\n",
       " (u'church.', 1),\n",
       " (u'fares', 2),\n",
       " (u'such-like', 1),\n",
       " (u'ordnance', 3),\n",
       " (u'fall,', 2),\n",
       " (u'exception', 1),\n",
       " (u'sponge!', 1),\n",
       " (u'patiently.', 1),\n",
       " (u'particular:', 1),\n",
       " (u'apprehension', 1),\n",
       " (u'decayer', 1),\n",
       " (u'Into', 4),\n",
       " (u'fall.', 1),\n",
       " (u'employ', 1),\n",
       " (u'enmity', 1),\n",
       " (u'parle,', 1),\n",
       " (u'unfellowed.', 1),\n",
       " (u'Call', 1),\n",
       " (u'cracked', 1),\n",
       " (u'leaves', 2),\n",
       " (u'skirts', 1),\n",
       " (u'seven', 1),\n",
       " (u'faith.', 2),\n",
       " (u'have', 159),\n",
       " (u'faith,', 6),\n",
       " (u'truster', 1),\n",
       " (u'is', 292),\n",
       " (u'defeated', 1),\n",
       " (u'faith:', 1),\n",
       " (u'need', 3),\n",
       " (u'penetrable', 1),\n",
       " (u'breath;', 1),\n",
       " (u'One.', 1),\n",
       " (u'exterior', 1),\n",
       " (u'consent:', 1),\n",
       " (u'Last', 1),\n",
       " (u'peak,', 1),\n",
       " (u'mouse', 1),\n",
       " (u'coming.', 1),\n",
       " (u'said.', 2),\n",
       " (u'grown', 4),\n",
       " (u'containing', 1),\n",
       " (u'Thoughts', 1),\n",
       " (u'heart.', 3),\n",
       " (u'heart,', 6),\n",
       " (u'make', 46),\n",
       " (u'Stop', 1),\n",
       " (u'heart:', 2),\n",
       " (u'Did', 13),\n",
       " (u'grapple', 1),\n",
       " (u'distempered.', 1),\n",
       " (u'liberal', 2),\n",
       " (u'swear.', 1),\n",
       " (u'heraldry,', 1),\n",
       " (u'wheel', 1),\n",
       " (u'youth', 10),\n",
       " (u'fust', 1),\n",
       " (u'discourse?', 1),\n",
       " (u'blue', 1),\n",
       " (u'hang', 3),\n",
       " (u'graveness.', 1),\n",
       " (u'evil', 1),\n",
       " (u'delight', 3),\n",
       " (u'must,', 1),\n",
       " (u'Carrying,', 1),\n",
       " (u'propose,', 1),\n",
       " (u'made,', 3),\n",
       " (u'thoughts', 12),\n",
       " (u'differences,', 1),\n",
       " (u'Mars,', 1),\n",
       " (u'Wittenberg.', 1),\n",
       " (u'ROSENCRANTZ', 57),\n",
       " (u'thy', 83),\n",
       " (u'shrewdly;', 1),\n",
       " (u'changes', 1),\n",
       " (u\"stay'd\", 2),\n",
       " (u'directly', 1),\n",
       " (u'visitation?', 1),\n",
       " (u'thing,', 4),\n",
       " (u\"in't\", 1),\n",
       " (u'means,', 4),\n",
       " (u'the', 995),\n",
       " (u'chamberlain.', 1),\n",
       " (u'merry?', 1),\n",
       " (u'givers', 1),\n",
       " (u'quoted', 1),\n",
       " (u'able', 1),\n",
       " (u'circumstance.', 1),\n",
       " (u'repose', 1),\n",
       " (u'proposer', 1),\n",
       " (u'behold!', 1),\n",
       " (u'rhymed.', 1),\n",
       " (u'admittance', 1),\n",
       " (u'[Pours', 1),\n",
       " (u'Sailors]', 1),\n",
       " (u'shrouding', 1),\n",
       " (u'wince,', 1),\n",
       " (u'ease', 2),\n",
       " (u'had', 38),\n",
       " (u'seem', 8),\n",
       " (u'reply:', 1),\n",
       " (u\"fall'n\", 1),\n",
       " (u\"ha't\", 1),\n",
       " (u'dreams', 2),\n",
       " (u'innocent', 1),\n",
       " (u'hap', 1),\n",
       " (u'pitiful', 1),\n",
       " (u'match', 1),\n",
       " (u'hat', 2),\n",
       " (u'worse.', 1),\n",
       " (u'Lord,', 2),\n",
       " (u'Herod:', 1),\n",
       " (u'desire', 4),\n",
       " (u'jealousy!', 1),\n",
       " (u'which', 36),\n",
       " (u'wall', 1),\n",
       " (u'trippingly', 1),\n",
       " (u'magic', 1),\n",
       " (u'sleep;', 2),\n",
       " (u'Try', 1),\n",
       " (u'ye;', 1),\n",
       " (u'honoured', 1),\n",
       " (u'mason,', 2),\n",
       " (u'birth', 2),\n",
       " (u'judge', 2),\n",
       " (u'feet]', 1),\n",
       " (u'shadow', 1),\n",
       " (u'burnt', 1),\n",
       " (u'opinion,', 1),\n",
       " (u'shoulder', 2),\n",
       " (u'soever', 1),\n",
       " (u'satisfaction;', 1),\n",
       " (u'ha,', 1),\n",
       " (u'throat;', 1),\n",
       " (u'dismay.', 1),\n",
       " (u'dangerous,', 1),\n",
       " (u'hill:', 1),\n",
       " (u'prating', 1),\n",
       " (u'tame,', 2),\n",
       " (u'private', 1),\n",
       " (u'hung', 1),\n",
       " (u'collection;', 1),\n",
       " (u'Witness', 1),\n",
       " (u'Now,', 8),\n",
       " (u'old', 18),\n",
       " (u'drowned', 2),\n",
       " (u'deal', 1),\n",
       " (u'Voltimand,', 2),\n",
       " (u'country.', 1),\n",
       " (u'Divided', 1),\n",
       " (u'crown', 1),\n",
       " (u'Methought', 1),\n",
       " (u'dead', 16),\n",
       " (u'prologue,', 1),\n",
       " (u'commission:', 1),\n",
       " (u\"nothing's\", 1),\n",
       " (u\"proclaim'd\", 1),\n",
       " (u'brief:', 1),\n",
       " (u'bore', 4),\n",
       " (u'unvalued', 1),\n",
       " (u\"on't\", 1),\n",
       " (u'ungracious', 1),\n",
       " (u'month:', 1),\n",
       " (u'perchance', 3),\n",
       " (u'voice:', 1),\n",
       " (u'for', 164),\n",
       " (u'Excellent,', 1),\n",
       " (u'bold,', 2),\n",
       " (u'evidence.', 1),\n",
       " (u'fencing,', 1),\n",
       " (u'sphere,', 1),\n",
       " (u'burns,', 2),\n",
       " (u'month,', 2),\n",
       " (u'rareness,', 1),\n",
       " (u'beget', 1),\n",
       " (u'purchases,', 1),\n",
       " (u'Calls', 1),\n",
       " (u\"'Thou\", 1),\n",
       " (u\"poison'd.\", 1),\n",
       " (u'fame,', 1),\n",
       " (u'ply', 1),\n",
       " (u'sift', 1),\n",
       " (u'suits,', 1),\n",
       " (u'paid', 1),\n",
       " (u'encompassment', 1),\n",
       " (u'Prison,', 1),\n",
       " (u'post', 1),\n",
       " (u\"to't;\", 1),\n",
       " (u'where?', 2),\n",
       " (u'Nay,', 21),\n",
       " (u'musty.', 1),\n",
       " (u'Indeed,', 6),\n",
       " (u'Rashly,', 1),\n",
       " (u'promontory,', 1),\n",
       " (u'table.', 1),\n",
       " (u'accepts', 1),\n",
       " (u'afternoon,', 1),\n",
       " (u'[Flourish', 1),\n",
       " (u'quite,', 1),\n",
       " (u\"'And\", 1),\n",
       " (u'twain!', 1),\n",
       " (u'majestical,', 1),\n",
       " (u'Sense,', 1),\n",
       " (u'incorpsed', 1),\n",
       " (u'treble', 2),\n",
       " (u'antique', 2),\n",
       " (u'Use', 1),\n",
       " (u'lief', 1),\n",
       " (u'off]', 1),\n",
       " (u'Make', 9),\n",
       " (u'Laertes.', 3),\n",
       " (u'for;', 1),\n",
       " (u\"poison'd:\", 1),\n",
       " (u'snow,', 3),\n",
       " (u'for?', 2),\n",
       " (u'others.', 1),\n",
       " (u'wrap', 1),\n",
       " (u'sultry,', 1),\n",
       " (u\"Saviour's\", 1),\n",
       " (u'quickness:', 1),\n",
       " (u'unseal', 1),\n",
       " (u'Unless', 1),\n",
       " (u'familiar,', 1),\n",
       " (u'wag', 1),\n",
       " (u'altitude', 1),\n",
       " (u'humble,', 1),\n",
       " (u'Refrain', 1),\n",
       " (u'Break', 3),\n",
       " (u'sends', 4),\n",
       " (u'bodykins,', 1),\n",
       " (u'profoundest', 1),\n",
       " (u'prevent,', 1),\n",
       " (u'brave,', 1),\n",
       " (u'music', 2),\n",
       " (u'was', 68),\n",
       " (u'wrote', 1),\n",
       " (u'questionable', 1),\n",
       " (u'head', 17),\n",
       " (u'form', 10),\n",
       " (u'VOLTIMAND,', 2),\n",
       " (u'anon;', 1),\n",
       " (u'visitation', 2),\n",
       " (u\"hallow'd\", 1),\n",
       " (u'qualifies', 1),\n",
       " (u'right,', 2),\n",
       " (u'heat', 2),\n",
       " (u'uncle', 4),\n",
       " (u'Been', 2),\n",
       " (u'pay,', 1),\n",
       " (u'babe!', 1),\n",
       " (u'so;', 2),\n",
       " (u'true', 14),\n",
       " (u'hands.', 1),\n",
       " (u'admiration', 1),\n",
       " (u'thing.', 1),\n",
       " (u'ignorance.', 1),\n",
       " (u'crew.', 1),\n",
       " (u'lips', 1),\n",
       " (u\"for's\", 2),\n",
       " (u'Cornelius,', 1),\n",
       " (u'again.', 8),\n",
       " (u'hall:', 2),\n",
       " (u'bilboes.', 1),\n",
       " (u'fools!', 1),\n",
       " (u'Messenger', 2),\n",
       " (u'alone;', 1),\n",
       " (u'die:', 1),\n",
       " (u'breathe', 3),\n",
       " (u'discovery,', 1),\n",
       " (u'sight', 3),\n",
       " (u'rendezvous.', 1),\n",
       " (u'stronger', 4),\n",
       " (u'abstract', 1),\n",
       " (u'dear', 27),\n",
       " (u'easing.', 1),\n",
       " (u'die,', 3),\n",
       " (u'brother.', 4),\n",
       " (u'door', 2),\n",
       " (u'awhile', 3),\n",
       " (u'left.', 1),\n",
       " (u'willingly', 2),\n",
       " (u'looked', 1),\n",
       " (u'Wretched', 1),\n",
       " (u'outstretched', 1),\n",
       " (u'woodcocks.', 1),\n",
       " (u'vanquisher;', 1),\n",
       " (u'rhapsody', 1),\n",
       " (u'when', 28),\n",
       " (u\"look'd\", 3),\n",
       " (u'Running', 1),\n",
       " (u\"o'ertook\", 1),\n",
       " (u'law:', 1),\n",
       " (u'bed;', 3),\n",
       " (u'unwilling', 1),\n",
       " (u'superfluous', 1),\n",
       " (u'Scripture?', 1),\n",
       " (u'life', 12),\n",
       " (u'denies', 1),\n",
       " (u'seeming-virtuous', 1),\n",
       " (u'intend', 1),\n",
       " (u'garland', 1),\n",
       " (u'preceding', 1),\n",
       " (u'stick:', 1),\n",
       " (u\"pin's\", 1),\n",
       " (u\"let's\", 6),\n",
       " (u'herb-grace', 1),\n",
       " (u'treads,', 1),\n",
       " (u'intent', 1),\n",
       " (u'portraiture', 1),\n",
       " (u\"'a\", 1),\n",
       " (u'variable', 2),\n",
       " (u'absurd:', 1),\n",
       " (u'died', 2),\n",
       " (u'feeling,', 1),\n",
       " (u'younger', 1),\n",
       " (u'Madness', 1),\n",
       " (u'diadem', 2),\n",
       " (u'assume', 3),\n",
       " (u'painting', 1),\n",
       " (u\"To't\", 1),\n",
       " (u'methinks.', 1),\n",
       " (u'daily', 1),\n",
       " (u'CORNELIUS,', 3),\n",
       " (u\"life's\", 1),\n",
       " (u'Second', 13),\n",
       " (u\"'A\", 1),\n",
       " (u'help,', 4),\n",
       " (u\"'O\", 1),\n",
       " (u'dust', 3),\n",
       " (u'good:', 2),\n",
       " (u\"'I\", 3),\n",
       " (u\"passion's\", 1),\n",
       " (u'customary', 1),\n",
       " (u'pictures,', 1),\n",
       " (u'stay,', 1),\n",
       " (u'peruse', 1),\n",
       " (u'leads', 1),\n",
       " (u'brow;', 1),\n",
       " (u'solicited.', 1),\n",
       " (u'ROSENCRANTZ]', 1),\n",
       " (u'honour.', 3),\n",
       " (u'garlands', 1),\n",
       " (u\"'High\", 1),\n",
       " (u'tenable', 1),\n",
       " (u'varnish', 1),\n",
       " (u'praises', 1),\n",
       " ...]"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "wordCounts = textFile.flatMap(lambda line: line.split()).map(lambda word: (word, 1)).reduceByKey(lambda a, b: a+b)\n",
    "wordCounts.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Using the Cache\n",
    "Pull a data sets into a cluster-wide in-memory cache. \n",
    "\n",
    "Very useful when using iterative algorithms.\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "data/books-eng/hamlet.txt MapPartitionsRDD[1] at textFile at NativeMethodAccessorImpl.java:0"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "textFile.cache()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Find words that start with an 'H' or 'h'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(u'heavens,', 2),\n",
       " (u'Hercules:', 1),\n",
       " (u'honour.', 3),\n",
       " (u'honour,', 2),\n",
       " (u'hand,', 3),\n",
       " (u'hand.', 2),\n",
       " (u'Had', 7),\n",
       " (u\"Hamlet.'\", 1),\n",
       " (u'Hide', 1),\n",
       " (u'hate', 1),\n",
       " (u'hole', 1),\n",
       " (u'happiness', 1),\n",
       " (u'him.', 27),\n",
       " (u'hebenon', 1),\n",
       " (u'herself', 5),\n",
       " (u'had', 38),\n",
       " (u'hot', 5),\n",
       " (u'Here', 6),\n",
       " (u'Honest,', 1),\n",
       " (u'hey-day', 1),\n",
       " (u'hither?', 2),\n",
       " (u'hands.', 1),\n",
       " (u'Hast', 1),\n",
       " (u'honoured', 1),\n",
       " (u'hammers', 1),\n",
       " (u'hat', 2),\n",
       " (u'hope', 5),\n",
       " (u'height,', 1),\n",
       " (u'Ha,', 1),\n",
       " (u'Hamlet:', 5),\n",
       " (u'history,', 1),\n",
       " (u'history.', 1),\n",
       " (u'Hamlet,', 25),\n",
       " (u'Hamlet.', 11),\n",
       " (u'hid', 1),\n",
       " (u'howling.', 1),\n",
       " (u'hedge', 1),\n",
       " (u'harlot', 1),\n",
       " (u'ha,', 1),\n",
       " (u'hypocrites;', 1),\n",
       " (u'hinges', 1),\n",
       " (u'hill:', 1),\n",
       " (u'hangers.', 1),\n",
       " (u'house.]', 2),\n",
       " (u'how,', 1),\n",
       " (u'heed', 2),\n",
       " (u'heraldry,', 1),\n",
       " (u'hung', 1),\n",
       " (u'help,', 4),\n",
       " (u'husband', 6),\n",
       " (u'heavy', 4),\n",
       " (u'havior,', 1),\n",
       " (u'hurt.', 1),\n",
       " (u'house.', 1),\n",
       " (u'house,', 1),\n",
       " (u'hair', 1),\n",
       " (u'hundred.', 1),\n",
       " (u'her?', 3),\n",
       " (u'holds,', 1),\n",
       " (u'highly,', 1),\n",
       " (u'heartily;', 1),\n",
       " (u'him:', 9),\n",
       " (u'horrid', 2),\n",
       " (u'haunt,', 1),\n",
       " (u\"Hyperion's\", 1),\n",
       " (u'hit;', 1),\n",
       " (u\"Hecate's\", 1),\n",
       " (u'HAMLET,', 4),\n",
       " (u'him,', 29),\n",
       " (u'health', 5),\n",
       " (u'herald', 1),\n",
       " (u'heavenly', 2),\n",
       " (u'helps,', 1),\n",
       " (u'honours.', 1),\n",
       " (u'Ho,', 2),\n",
       " (u\"Here's\", 4),\n",
       " (u'hap', 1),\n",
       " (u'headshake,', 1),\n",
       " (u'Hecuba', 1),\n",
       " (u'here', 27),\n",
       " (u'honest,', 1),\n",
       " (u'honest.', 1),\n",
       " (u'hectic', 1),\n",
       " (u'hast', 14),\n",
       " (u'hat]', 1),\n",
       " (u'harshly', 1),\n",
       " (u'hoops', 1),\n",
       " (u'he.', 1),\n",
       " (u'hours', 1),\n",
       " (u'herself;', 1),\n",
       " (u'Happy,', 1),\n",
       " (u'hearing,', 2),\n",
       " (u'his,', 1),\n",
       " (u'himself.', 4),\n",
       " (u'host', 1),\n",
       " (u'Hail', 1),\n",
       " (u'handsaw.', 1),\n",
       " (u'home:', 2),\n",
       " (u\"honour'd\", 3),\n",
       " (u'her;', 1),\n",
       " (u'heel', 1),\n",
       " (u'Hecuba!', 1),\n",
       " (u'Haste', 1),\n",
       " (u'here!', 2),\n",
       " (u'hams:', 1),\n",
       " (u'himself:', 1),\n",
       " (u'hither', 2),\n",
       " (u'heedful', 1),\n",
       " (u'here?', 2),\n",
       " (u'Her', 6),\n",
       " (u'hellish', 1),\n",
       " (u\"Howe'er\", 1),\n",
       " (u'hope!', 1),\n",
       " (u'have?', 2),\n",
       " (u'heaven-kissing', 1),\n",
       " (u'hangers,', 1),\n",
       " (u'humourous', 1),\n",
       " (u'heathen?', 1),\n",
       " (u'heroes', 1),\n",
       " (u'Hymen', 1),\n",
       " (u'her', 70),\n",
       " (u'Hoist', 1),\n",
       " (u'Happily', 1),\n",
       " (u'Hyperion', 1),\n",
       " (u'humble,', 1),\n",
       " (u'Hazard', 1),\n",
       " (u'Herod:', 1),\n",
       " (u'hobby-horse,', 1),\n",
       " (u'highness:', 1),\n",
       " (u\"ha't\", 1),\n",
       " (u'hardly', 1),\n",
       " (u'home,', 1),\n",
       " (u\"harlot's\", 1),\n",
       " (u'head', 17),\n",
       " (u'holy', 2),\n",
       " (u'healthful', 1),\n",
       " (u'heir', 1),\n",
       " (u'hits', 1),\n",
       " (u\"hallow'd\", 1),\n",
       " (u'heartily', 1),\n",
       " (u'heat', 2),\n",
       " (u'hear', 23),\n",
       " (u'hath,', 2),\n",
       " (u'Hark', 1),\n",
       " (u'He.', 1),\n",
       " (u'hands,', 3),\n",
       " (u\"here's\", 4),\n",
       " (u'hush', 1),\n",
       " (u'hire', 1),\n",
       " (u'Hear', 1),\n",
       " (u'hour', 5),\n",
       " (u'hideous', 1),\n",
       " (u'head;', 1),\n",
       " (u\"HAMLET.'\", 2),\n",
       " (u'himself,', 2),\n",
       " (u'harping', 1),\n",
       " (u'hall:', 2),\n",
       " (u'he,', 7),\n",
       " (u'heart-ache', 1),\n",
       " (u'Hillo,', 2),\n",
       " (u'Horatio,', 13),\n",
       " (u'horses', 1),\n",
       " (u'Horatio.', 5),\n",
       " (u'have', 159),\n",
       " (u'hears', 3),\n",
       " (u'hideous;', 1),\n",
       " (u'honey', 1),\n",
       " (u'heels', 3),\n",
       " (u'half.', 1),\n",
       " (u'Horatio:', 2),\n",
       " (u'husband;', 1),\n",
       " (u'heaven,', 11),\n",
       " (u'hoar', 1),\n",
       " (u'hit', 2),\n",
       " (u'high', 2),\n",
       " (u'horseback:', 1),\n",
       " (u'heart.', 3),\n",
       " (u'heaven.', 3),\n",
       " (u'Holding', 1),\n",
       " (u'hilts,', 1),\n",
       " (u'heart:', 2),\n",
       " (u'husbandry.', 1),\n",
       " (u'hides', 1),\n",
       " (u'honourable.', 1),\n",
       " (u'haste', 8),\n",
       " (u'how.', 1),\n",
       " (u'Help,', 2),\n",
       " (u'hell,', 5),\n",
       " (u'HORATIO,', 7),\n",
       " (u'Have', 16),\n",
       " (u'hear?', 2),\n",
       " (u'hang', 3),\n",
       " (u'hent:', 1),\n",
       " (u'hence', 3),\n",
       " (u'Hold,', 1),\n",
       " (u'hide', 3),\n",
       " (u'hearer:', 1),\n",
       " (u'hems,', 1),\n",
       " (u\"husband's\", 2),\n",
       " (u'hide?', 1),\n",
       " (u\"hold'st\", 1),\n",
       " (u'heart,', 6),\n",
       " (u'heard,', 5),\n",
       " (u'hasty', 1),\n",
       " (u\"have't.\", 1),\n",
       " (u'ho,', 4),\n",
       " (u'hands:', 1),\n",
       " (u'High', 1),\n",
       " (u'hold,', 3),\n",
       " (u'his:', 1),\n",
       " (u'Hyrcanian', 1),\n",
       " (u'herb-grace', 1),\n",
       " (u'hoped', 1),\n",
       " (u'heat,', 2),\n",
       " (u'help', 4),\n",
       " (u'heavily', 1),\n",
       " (u'heaven?', 2),\n",
       " (u'hits:', 1),\n",
       " (u'hell?', 1),\n",
       " (u'hath', 52),\n",
       " (u'Horatio;', 1),\n",
       " (u'higher', 1),\n",
       " (u'hell', 4),\n",
       " (u'houses', 1),\n",
       " (u'heavens', 3),\n",
       " (u'His', 17),\n",
       " (u\"he's\", 8),\n",
       " (u\"Honour'd,\", 1),\n",
       " (u'hall', 2),\n",
       " (u'honesty?', 1),\n",
       " (u'Hic', 1),\n",
       " (u'helps', 1),\n",
       " (u'hopes,', 1),\n",
       " (u'has', 15),\n",
       " (u'hearers', 1),\n",
       " (u'humanity', 1),\n",
       " (u'heart?', 2),\n",
       " (u'his', 279),\n",
       " (u'halt', 1),\n",
       " (u'Hamlet;', 3),\n",
       " (u'heaven!', 2),\n",
       " (u'HORATIO', 114),\n",
       " (u'hit.', 1),\n",
       " (u'heaven;', 5),\n",
       " (u'hands', 7),\n",
       " (u'hitherto', 2),\n",
       " (u'him', 118),\n",
       " (u'helpful', 1),\n",
       " (u'heaven', 16),\n",
       " (u'hereafter', 1),\n",
       " (u'him]', 1),\n",
       " (u'held', 2),\n",
       " (u'ha!', 2),\n",
       " (u'help!', 1),\n",
       " (u'hill;', 1),\n",
       " (u\"Hamlet's\", 8),\n",
       " (u'hollow', 1),\n",
       " (u'Hath', 8),\n",
       " (u'Hercules', 2),\n",
       " (u'ha?', 1),\n",
       " (u'Hum!', 1),\n",
       " (u'hasten', 1),\n",
       " (u'Horatio', 3),\n",
       " (u'hangers', 1),\n",
       " (u'honesty', 4),\n",
       " (u'her,', 6),\n",
       " (u'house', 1),\n",
       " (u'hard', 4),\n",
       " (u'hour.', 1),\n",
       " (u'hour,', 2),\n",
       " (u\"house's\", 1),\n",
       " (u'Heaven', 4),\n",
       " (u'honourable', 2),\n",
       " (u'hits.', 1),\n",
       " (u'home', 2),\n",
       " (u\"ha'\", 3),\n",
       " (u'hence:', 1),\n",
       " (u'horrors,', 1),\n",
       " (u\"heaven's\", 1),\n",
       " (u'him;', 7),\n",
       " (u'hypocrite,', 1),\n",
       " (u'hourly', 1),\n",
       " (u'highest', 1),\n",
       " (u'heavy-headed', 1),\n",
       " (u'Half', 1),\n",
       " (u'howsoever', 1),\n",
       " (u'habits', 1),\n",
       " (u'hill', 1),\n",
       " (u'horrible', 1),\n",
       " (u'Hold', 4),\n",
       " (u'hit,', 2),\n",
       " (u'him!', 4),\n",
       " (u'HAMLET]', 11),\n",
       " (u'honest?', 1),\n",
       " (u'heel,', 1),\n",
       " (u'honest;', 1),\n",
       " (u'Hamlets', 1),\n",
       " (u'halfpenny.', 1),\n",
       " (u'horses:', 1),\n",
       " (u'HAMLET', 367),\n",
       " (u'HORATIO]', 7),\n",
       " (u'Himself', 1),\n",
       " (u'Here,', 5),\n",
       " (u'hand', 13),\n",
       " (u'harsh;', 1),\n",
       " (u\"HAMLET's]\", 1),\n",
       " (u'head,', 2),\n",
       " (u'heavy,', 2),\n",
       " (u'hearsed', 1),\n",
       " (u'havoc.', 1),\n",
       " (u'hither,', 5),\n",
       " (u'hid,', 1),\n",
       " (u'hugger-mugger', 1),\n",
       " (u'honest', 6),\n",
       " (u'here.', 2),\n",
       " (u'here,', 10),\n",
       " (u'how?', 1),\n",
       " (u'happen.', 1),\n",
       " (u'himself;', 4),\n",
       " (u'hand?', 1),\n",
       " (u'Hadst', 1),\n",
       " (u'hope,', 2),\n",
       " (u'hurts', 1),\n",
       " (u'Hey', 1),\n",
       " (u'hearing', 3),\n",
       " (u'hardy', 1),\n",
       " (u'have.', 1),\n",
       " (u'have,', 3),\n",
       " (u'hear.', 5),\n",
       " (u'HAMLET;', 1),\n",
       " (u'habit', 4),\n",
       " (u'hey', 1),\n",
       " (u'horse,', 3),\n",
       " (u'home!', 1),\n",
       " (u'Ha!', 2),\n",
       " (u'husband.', 1),\n",
       " (u'hundred', 1),\n",
       " (u'heartily.', 1),\n",
       " (u'hatchment', 1),\n",
       " (u'happy', 2),\n",
       " (u'hue', 1),\n",
       " (u'himself', 9),\n",
       " (u'hover', 1),\n",
       " (u'homage', 1),\n",
       " (u'Hecuba,', 1),\n",
       " (u'hoodman-blind?', 1),\n",
       " (u'Having', 1),\n",
       " (u'herein', 3),\n",
       " (u'hurt', 1),\n",
       " (u'hard;', 1),\n",
       " (u'heaves:', 1),\n",
       " (u'horridly', 2),\n",
       " (u'hold', 21),\n",
       " (u'herein;', 1),\n",
       " (u'he', 168),\n",
       " (u'haste.', 3),\n",
       " (u'How', 56),\n",
       " (u'he?', 2),\n",
       " (u'head:', 2),\n",
       " (u'Hamlet', 23),\n",
       " (u'hang,', 1),\n",
       " (u'Horatio?', 1),\n",
       " (u'head.', 3),\n",
       " (u'historical-pastoral,', 1),\n",
       " (u'heavens!', 3),\n",
       " (u'harrow', 1),\n",
       " (u'hair,', 1),\n",
       " (u'heart', 11),\n",
       " (u'haply', 1),\n",
       " (u\"He's\", 2),\n",
       " (u'happily,', 1),\n",
       " (u'half', 3),\n",
       " (u'Holds', 1),\n",
       " (u'Horatio!', 5),\n",
       " (u'heard', 9),\n",
       " (u'Hamlet?', 4),\n",
       " (u'hours.', 1),\n",
       " (u'He', 39),\n",
       " (u'heads:', 1),\n",
       " (u'her.', 5),\n",
       " (u'huge', 1),\n",
       " (u'handsome', 1),\n",
       " (u'hart', 1),\n",
       " (u'harrows', 1),\n",
       " (u'holds', 3),\n",
       " (u'hearers?', 1),\n",
       " (u'Holla!', 1),\n",
       " (u'haps,', 1),\n",
       " (u'how', 31),\n",
       " (u\"honour's\", 1),\n",
       " (u\"heart's\", 1),\n",
       " (u'hatch', 1),\n",
       " (u'health.', 1),\n",
       " (u'hast,', 1),\n",
       " (u'hies', 1),\n",
       " (u'heraldry', 1),\n",
       " (u'hadst', 2),\n",
       " (u'husbands.', 1),\n",
       " (u'him?', 2),\n",
       " (u'harsh', 1),\n",
       " (u'Hecuba.', 1),\n",
       " (u'hear:', 1),\n",
       " (u'Has', 1),\n",
       " (u'horrible!', 3),\n",
       " (u'Ho!', 1),\n",
       " (u'hobby-horse', 1),\n",
       " (u'hear,', 1),\n",
       " (u'honour', 5),\n",
       " (u'hawk', 1),\n",
       " (u\"he'll\", 1),\n",
       " (u'harmony;', 1),\n",
       " (u\"ha't.\", 1),\n",
       " (u'ho!', 7),\n",
       " (u'honeying', 1),\n",
       " (u'Hamlet!', 6),\n",
       " (u'hot.', 1),\n",
       " (u'heart;', 5),\n",
       " (u'harbingers', 1),\n",
       " (u'humbly', 5),\n",
       " (u'hearts', 2),\n",
       " (u'Hunts', 1),\n",
       " (u'Heavens', 1),\n",
       " (u'having', 1),\n",
       " (u'Haply', 1)]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "words = textFile.flatMap(lambda line: line.split())\n",
    "words_subset = words.filter(lambda x: x[0] in ['H','h'])\n",
    "counts = words_subset.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "counts.collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Cleanup using a regular expression\n",
    "\n",
    "* remove all non-alphanumeric characters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "affirme 1\n",
      "alternatives 1\n",
      "atomic 1\n",
      "ALVAH 1\n",
      "Anna 293\n",
      "appealed 3\n",
      "apace 8\n",
      "apes 3\n",
      "Archelaus 1\n",
      "attirer 1\n",
      "Apparition 7\n",
      "assassination 2\n",
      "appelez 1\n",
      "add 18\n",
      "ALEXAS 24\n",
      "adequate 2\n",
      "associations 3\n",
      "accessary 1\n",
      "amorous 9\n",
      "adjacent 6\n",
      "Arnauts 2\n",
      "Architect 4\n",
      "ambition 23\n",
      "affect 8\n",
      "annoncez 1\n",
      "administer 2\n",
      "Art 24\n",
      "attentively 32\n",
      "ascertaining 1\n",
      "augmenting 1\n",
      "avenged 3\n",
      "architecture 1\n",
      "attendre 3\n",
      "adorers 1\n",
      "audible 19\n",
      "Accursed 2\n",
      "antique 12\n",
      "Adding 1\n",
      "autorité 1\n",
      "Army 13\n",
      "arracher 3\n",
      "accord 29\n",
      "across 194\n",
      "Accountant 2\n",
      "assoupissantes 1\n",
      "appear 94\n",
      "Aline 2\n",
      "Articulation 2\n",
      "arrivance 1\n",
      "appals 1\n",
      "Aurora 1\n",
      "astray 5\n",
      "attracted 21\n",
      "abides 1\n",
      "Armenia 2\n",
      "APPROACH 1\n",
      "admiral 5\n",
      "Animators 3\n",
      "available 12\n",
      "accordent 1\n",
      "Alexey 1\n",
      "abnormally 1\n",
      "alone 325\n",
      "article 11\n",
      "Abhor 1\n",
      "accident 18\n",
      "Aerial 2\n",
      "attacks 18\n",
      "Ahh 1\n",
      "against 359\n",
      "agite 1\n",
      "anime 1\n",
      "Alsatian 2\n",
      "adjutants 37\n",
      "Anvil 2\n",
      "appeared 128\n",
      "acted 28\n",
      "accorded 1\n",
      "Akharovs 1\n",
      "accommodate 1\n",
      "apply 13\n",
      "Admirable 1\n",
      "assortment 1\n",
      "Amen 14\n",
      "aligning 1\n",
      "aristocratic 3\n",
      "avais 17\n",
      "aperçois 1\n",
      "airs 7\n",
      "ANDY 1\n",
      "Arranging 1\n",
      "armchairs 2\n",
      "Anoat 2\n",
      "Aaagh 1\n",
      "agonizingly 1\n",
      "abrégent 1\n",
      "arises 8\n",
      "appositeness 1\n",
      "aurons 1\n",
      "astro 8\n",
      "ABOVE 1\n",
      "authors 4\n",
      "apologizing 2\n",
      "alderman 1\n",
      "argent 14\n",
      "Approaching 1\n",
      "affrighted 3\n",
      "allurement 1\n",
      "atmospheric 1\n",
      "advocating 1\n",
      "artisans 4\n",
      "aims 28\n",
      "anywhere 28\n",
      "adore 4\n",
      "alloit 1\n",
      "anchor 3\n",
      "alarment 1\n",
      "arriva 1\n",
      "accentuated 1\n",
      "Asks 1\n",
      "appartement 2\n",
      "attaché 2\n",
      "ages 9\n",
      "autumn 31\n",
      "au 145\n",
      "Announce 2\n",
      "awhile 47\n",
      "attract 13\n",
      "aventures 2\n",
      "Asp 1\n",
      "affined 2\n",
      "attaqué 1\n",
      "also 289\n",
      "Asch 2\n",
      "aching 5\n",
      "assailing 1\n",
      "added 217\n",
      "aspen 3\n",
      "Always 11\n",
      "assist 8\n",
      "Alep 1\n",
      "Avant 1\n",
      "Anchorhead 3\n",
      "alarms 3\n",
      "Abuses 1\n",
      "awe 18\n",
      "arshin 1\n",
      "armrest 1\n",
      "afresh 9\n",
      "appellations 1\n",
      "applause 4\n",
      "age 87\n",
      "apathy 2\n",
      "anybody 16\n",
      "admiringly 1\n",
      "absurd 16\n",
      "Ay 139\n",
      "abroach 1\n",
      "axles 1\n",
      "animés 2\n",
      "augurer 1\n",
      "avidité 1\n",
      "appareil 1\n",
      "affright 3\n",
      "attempts 18\n",
      "Agitated 2\n",
      "amiss 15\n",
      "asylum 6\n",
      "arrogant 2\n",
      "apprendrons 1\n",
      "Alexanders 2\n",
      "attires 2\n",
      "answered 157\n",
      "advanced 60\n",
      "aimais 3\n",
      "arrêtés 1\n",
      "announce 10\n",
      "abri 1\n",
      "ayez 8\n",
      "assignment 1\n",
      "aigrisse 1\n",
      "asseoir 1\n",
      "amulet 1\n",
      "Assure 1\n",
      "avarice 2\n",
      "Adversity 1\n",
      "areas 1\n",
      "acquainted 23\n",
      "Armored 1\n",
      "artist 2\n",
      "Auntie 2\n",
      "ashen 1\n",
      "appropriate 4\n",
      "absurdly 1\n",
      "Amphilochus 1\n",
      "Anatole 222\n",
      "Advancing 2\n",
      "alms 5\n",
      "Austrian 69\n",
      "ACTE 10\n",
      "ankles 3\n",
      "Anticipation 1\n",
      "Avec 7\n",
      "abundantly 1\n",
      "accusez 1\n",
      "Apparently 1\n",
      "Apercevant 1\n",
      "ardent 6\n",
      "ambassadors 7\n",
      "aroused 30\n",
      "avert 3\n",
      "attitudes 1\n",
      "ASHLEY 1\n",
      "avoid 70\n",
      "attentats 1\n",
      "atteintes 1\n",
      "adhesion 1\n",
      "accomplie 2\n",
      "assuming 6\n",
      "Abandoning 1\n",
      "Applying 2\n",
      "anthropophages 1\n",
      "accustom 4\n",
      "Age 2\n",
      "Auditor 1\n",
      "acclaimed 1\n",
      "animating 1\n",
      "AND 18\n",
      "aslant 1\n",
      "ahead 49\n",
      "aima 1\n",
      "accepts 3\n",
      "Att 1\n",
      "accomplished 20\n",
      "aboard 18\n",
      "Armfeldt 12\n",
      "Alesha 1\n",
      "among 221\n",
      "adjusted 9\n",
      "already 371\n",
      "agitée 1\n",
      "awfully 5\n",
      "Add 1\n",
      "Assistants 9\n",
      "ancre 1\n",
      "Approach 5\n",
      "apprehended 1\n",
      "applaudissait 1\n",
      "Abashed 1\n",
      "arcade 1\n",
      "allude 1\n",
      "Assistant 31\n",
      "affranchis 1\n",
      "ardents 1\n",
      "anyone 203\n",
      "Ajax 1\n",
      "aspersion 1\n",
      "analyze 3\n",
      "Armenian 6\n",
      "attacked 32\n",
      "autumnal 1\n",
      "Auditors 1\n",
      "affetto 2\n",
      "ardently 1\n",
      "acknown 1\n",
      "Apocalypse 4\n",
      "aery 1\n",
      "arisen 3\n",
      "accouplent 1\n",
      "avoué 2\n",
      "aught 24\n",
      "affront 2\n",
      "anxieties 1\n",
      "abductors 1\n",
      "antipathies 1\n",
      "akin 6\n",
      "Apart 9\n",
      "amère 1\n",
      "applauses 1\n",
      "abattre 1\n",
      "amener 5\n",
      "accouchera 1\n",
      "antagonists 2\n",
      "amiability 3\n",
      "Agrippa 10\n",
      "ai 136\n",
      "Am 32\n",
      "accents 6\n",
      "agirai 1\n",
      "accept 25\n",
      "ALSUP 1\n",
      "associates 3\n",
      "at 5146\n",
      "assassinat 1\n",
      "Arkharovs 4\n",
      "ADMIRAL 2\n",
      "awed 3\n",
      "Atoms 1\n",
      "application 10\n",
      "adornings 1\n",
      "Azor 1\n",
      "auditor 3\n",
      "adieu 23\n",
      "amities 1\n",
      "Ackbar 10\n",
      "abdomen 2\n",
      "Avoue 1\n",
      "AS 2\n",
      "Accounted 1\n",
      "arrived 102\n",
      "Allow 13\n",
      "angerly 1\n",
      "avail 10\n",
      "accelerates 1\n",
      "almshouses 1\n",
      "Adraksin 4\n",
      "Austrians 19\n",
      "alarm 41\n",
      "acknowledged 7\n",
      "autorisé 1\n",
      "avancez 1\n",
      "Another 96\n",
      "amuses 2\n",
      "antidotes 1\n",
      "ambiguities 1\n",
      "augurs 1\n",
      "angelically 1\n",
      "abstain 2\n",
      "Author 2\n",
      "anyway 5\n",
      "awful 26\n",
      "Afraid 7\n",
      "aspired 1\n",
      "autorités 1\n",
      "abreast 5\n",
      "abstention 1\n",
      "anneau 1\n",
      "aggravate 1\n",
      "accablent 1\n",
      "ascribing 1\n",
      "Astro 1\n",
      "adressait 1\n",
      "absence 64\n",
      "astonishment 14\n",
      "alter 15\n",
      "achieve 5\n",
      "ADAM 2\n",
      "allumé 3\n",
      "amended 1\n",
      "Aristovo 2\n",
      "Aah 1\n",
      "adheres 1\n",
      "Adore 1\n",
      "accuse 7\n",
      "apporte 2\n",
      "accepted 41\n",
      "antagonist 1\n",
      "appearing 17\n",
      "anaudience 1\n",
      "accompanying 9\n",
      "asparagus 2\n",
      "adroitement 1\n",
      "adjust 4\n",
      "arraign 1\n",
      "assemblée 1\n",
      "armourer 1\n",
      "ATTENDANT 1\n",
      "actuellement 1\n",
      "Aroused 1\n",
      "afforded 5\n",
      "allusion 2\n",
      "accelerator 1\n",
      "APRIL 1\n",
      "agricultural 1\n",
      "angina 4\n",
      "allay 4\n",
      "arm 246\n",
      "agit 4\n",
      "album 4\n",
      "announces 2\n",
      "anniversary 1\n",
      "Arc 1\n",
      "affords 5\n",
      "achievements 2\n",
      "assembly 6\n",
      "approached 70\n",
      "AHEAD 1\n",
      "admits 3\n",
      "approaches 27\n",
      "accommodated 1\n",
      "Addicted 1\n",
      "aposté 1\n",
      "amer 1\n",
      "adversary 3\n",
      "adulterate 2\n",
      "ambler 1\n",
      "asunder 2\n",
      "approbation 2\n",
      "appointments 5\n",
      "alighted 2\n",
      "Awoke 1\n",
      "Arabia 2\n",
      "attributes 3\n",
      "accepte 1\n",
      "attrait 1\n",
      "appropriately 1\n",
      "amassez 1\n",
      "adders 1\n",
      "abstiens 1\n",
      "allegiance 5\n",
      "apt 16\n",
      "AMUNDSON 1\n",
      "Also 1\n",
      "accomplishments 1\n",
      "adjoining 14\n",
      "along 289\n",
      "armure 1\n",
      "allait 1\n",
      "adresser 2\n",
      "affably 1\n",
      "amaze 3\n",
      "Attempted 1\n",
      "angry 143\n",
      "autocrat 1\n",
      "Admitting 1\n",
      "ah 27\n",
      "Assemble 2\n",
      "assez 17\n",
      "Arrange 1\n",
      "accompany 10\n",
      "acceptable 2\n",
      "amity 3\n",
      "Adored 1\n",
      "avancé 1\n",
      "Alderaan 27\n",
      "apprentice 3\n",
      "armpits 3\n",
      "ancestral 1\n",
      "admire 4\n",
      "ain 7\n",
      "amitié 13\n",
      "animated 66\n",
      "avaient 3\n",
      "Alla 1\n",
      "attribute 6\n",
      "abbe 16\n",
      "arrangements 21\n",
      "anticipating 1\n",
      "Apollodorus 1\n",
      "aged 19\n",
      "anguish 7\n",
      "adopté 1\n",
      "awkwardness 9\n",
      "abjects 1\n",
      "administration 4\n",
      "arrêter 5\n",
      "accusations 3\n",
      "aprons 3\n",
      "advise 14\n",
      "ancestor 3\n",
      "Albeit 1\n",
      "absentees 1\n",
      "appwove 1\n",
      "assimilated 2\n",
      "advice 39\n",
      "Alexis 1\n",
      "afterburners 1\n",
      "averting 2\n",
      "Angina 1\n",
      "Alexas 10\n",
      "assise 1\n",
      "audit 5\n",
      "American 1\n",
      "artful 2\n",
      "amends 3\n",
      "artistic 2\n",
      "apparel 3\n",
      "augmented 2\n",
      "Apace 1\n",
      "Arraigning 1\n",
      "athletic 1\n",
      "applauding 1\n",
      "absurdité 1\n",
      "alien 30\n",
      "Ardente 1\n",
      "Across 2\n",
      "anticipation 4\n",
      "appliance 1\n",
      "Athwart 1\n",
      "affinity 4\n",
      "attained 17\n",
      "ailé 1\n",
      "autocratic 1\n",
      "assassinée 2\n",
      "Agrafena 6\n",
      "ANGUS 12\n",
      "affligeait 1\n",
      "anticipated 5\n",
      "antigravity 1\n",
      "Antichrist 5\n",
      "ACTUAL 2\n",
      "austerity 1\n",
      "adaptations 1\n",
      "acute 8\n",
      "appliquent 1\n",
      "assortis 1\n",
      "affordeth 1\n",
      "Ambassadors 2\n",
      "accusers 1\n",
      "assemblage 2\n",
      "Again 45\n",
      "atheistic 1\n",
      "abstiendrai 1\n",
      "art 218\n",
      "Augesd 5\n",
      "afflict 2\n",
      "auquel 1\n",
      "adducing 1\n",
      "arbitrating 1\n",
      "aloof 9\n",
      "activated 1\n",
      "aidera 1\n",
      "awaits 8\n",
      "Attendants 37\n",
      "Adieu 31\n",
      "alcohol 1\n",
      "Awakes 1\n",
      "afloat 3\n",
      "alders 1\n",
      "armadas 1\n",
      "Akhrosimova 3\n",
      "acting 20\n",
      "Alone 2\n",
      "acutely 3\n",
      "Apprentice 1\n",
      "arrival 55\n",
      "Apsheron 4\n",
      "aquaintances 1\n",
      "analyzed 2\n",
      "aspic 4\n",
      "amères 1\n",
      "abruptly 14\n",
      "assiduous 2\n",
      "adjusts 24\n",
      "ajouter 2\n",
      "achète 1\n",
      "admit 47\n",
      "absorption 2\n",
      "arbres 1\n",
      "Assisted 1\n",
      "armes 7\n",
      "accouche 1\n",
      "alleys 3\n",
      "aimed 12\n",
      "acquise 2\n",
      "acrid 1\n",
      "annoyance 13\n",
      "attaque 4\n",
      "attendance 17\n",
      "area 32\n",
      "assumed 51\n",
      "avenir 1\n",
      "apprehend 6\n",
      "Advisers 1\n",
      "annonce 2\n",
      "apprend 1\n",
      "acquisitions 1\n",
      "Allies 3\n",
      "apparent 16\n",
      "ANTEROOM 4\n",
      "Apparitions 2\n",
      "alas 15\n",
      "applauds 1\n",
      "Afterwards 7\n",
      "augment 1\n",
      "administrative 6\n",
      "attached 37\n",
      "Alive 1\n",
      "awakened 10\n",
      "Archive 26\n",
      "antinational 1\n",
      "agitation 45\n",
      "almost 219\n",
      "Acheron 1\n",
      "awesome 13\n",
      "alongside 5\n",
      "aussi 60\n",
      "adornment 1\n",
      "admonitory 1\n",
      "annoying 2\n",
      "advancing 29\n",
      "accommodera 1\n",
      "angle 12\n",
      "allurements 1\n",
      "ARBOGAST 3\n",
      "Adonis 1\n",
      "Arab 4\n",
      "accoutumés 1\n",
      "analogy 1\n",
      "affaire 13\n",
      "alienating 1\n",
      "armée 3\n",
      "android 1\n",
      "Auersperg 12\n",
      "Amelie 3\n",
      "argal 3\n",
      "Abused 1\n",
      "appellation 2\n",
      "aright 3\n",
      "arcs 1\n",
      "affeer 1\n",
      "accurately 5\n",
      "aloud 30\n",
      "affirmatively 4\n",
      "Alms 1\n",
      "ago 83\n",
      "Arrachez 1\n",
      "appropriated 2\n",
      "austere 2\n",
      "Arinka 1\n",
      "amie 5\n",
      "attracts 2\n",
      "accomplishing 1\n",
      "appreciate 7\n",
      "alarme 2\n",
      "appalled 3\n",
      "ANNICK 1\n",
      "arrivé 11\n",
      "alright 2\n",
      "amusements 3\n",
      "annoyed 7\n",
      "arousing 6\n",
      "ambassador 18\n",
      "Ancient 3\n",
      "Allumez 1\n",
      "Acts 2\n",
      "arrests 1\n",
      "angelic 2\n",
      "appetite 11\n",
      "Assez 1\n",
      "adding 11\n",
      "allumer 1\n",
      "assented 11\n",
      "attelle 1\n",
      "arrest 9\n",
      "award 1\n",
      "anterooms 1\n",
      "auprès 12\n",
      "acquitte 1\n",
      "Attendez 1\n",
      "awareness 1\n",
      "agriculturists 1\n",
      "around 332\n",
      "assignat 1\n",
      "availed 1\n",
      "Aunt 18\n",
      "Anakin 7\n",
      "assis 2\n",
      "aspirations 3\n",
      "artery 1\n",
      "abacus 1\n",
      "arched 4\n",
      "apprendre 4\n",
      "articulated 1\n",
      "abîmes 1\n",
      "alacrity 1\n",
      "aucun 14\n",
      "accueillie 1\n",
      "Alenina 1\n",
      "alternately 3\n",
      "applicable 10\n",
      "ANCHORHEAD 3\n",
      "ambling 2\n",
      "Are 129\n",
      "arrow 4\n",
      "avantage 2\n",
      "auld 1\n",
      "accessible 5\n",
      "adobe 1\n",
      "attachments 3\n",
      "Attila 1\n",
      "Aussi 2\n",
      "Avoid 1\n",
      "ascend 2\n",
      "alive 70\n",
      "abler 1\n",
      "autre 42\n",
      "altars 1\n",
      "Assumption 4\n",
      "abominable 7\n",
      "absurdity 4\n",
      "aptly 1\n",
      "atteint 2\n",
      "astrodroid 2\n",
      "Ahead 2\n",
      "adieus 1\n",
      "away 920\n",
      "anecdotes 5\n",
      "Accoutred 1\n",
      "Attending 1\n",
      "aggrandizement 4\n",
      "Abate 1\n",
      "aflame 1\n",
      "Alarm 1\n",
      "appartient 3\n",
      "alleged 5\n",
      "Apology 1\n",
      "associate 3\n",
      "arrivèrent 1\n",
      "According 10\n",
      "ascending 6\n",
      "assurer 2\n",
      "auteur 2\n",
      "address 34\n",
      "accusé 2\n",
      "angling 3\n",
      "appears 70\n",
      "Angeles 1\n",
      "actual 17\n",
      "arrivera 2\n",
      "accablé 1\n",
      "attributed 14\n",
      "aura 5\n",
      "aveugles 1\n",
      "Aaah 1\n",
      "adopting 3\n",
      "ANY 6\n",
      "AUTREY 1\n",
      "agisses 1\n",
      "attractions 2\n",
      "atomies 1\n",
      "apprehensive 3\n",
      "ALAN 4\n",
      "attentat 1\n",
      "affreux 1\n",
      "Asking 1\n",
      "amidst 1\n",
      "awoke 16\n",
      "armada 4\n",
      "auditing 1\n",
      "attendant 13\n",
      "accursed 3\n",
      "aimlessly 2\n",
      "agreeably 4\n",
      "aren 23\n",
      "AVEC 1\n",
      "alarums 2\n",
      "arms 223\n",
      "ARNOLD 1\n",
      "attaching 3\n",
      "Alcides 1\n",
      "angels 7\n",
      "autres 21\n",
      "adopts 1\n",
      "approchant 1\n",
      "artillery 53\n",
      "assailed 3\n",
      "ascribe 4\n",
      "arrête 2\n",
      "angles 2\n",
      "atom 3\n",
      "aide 55\n",
      "Alexandre 3\n",
      "Augurs 1\n",
      "anxiety 25\n",
      "auras 1\n",
      "announcing 7\n",
      "anyhow 6\n",
      "anticipate 3\n",
      "années 1\n",
      "appendage 2\n",
      "amuse 8\n",
      "aurai 7\n",
      "ANGLE 1\n",
      "ascends 1\n",
      "afflige 1\n",
      "agent 5\n",
      "Attack 2\n",
      "admires 1\n",
      "ay 18\n",
      "alliances 4\n",
      "absolute 27\n",
      "Axes 1\n",
      "awkward 30\n",
      "administered 4\n",
      "actor 9\n",
      "adequately 1\n",
      "actors 6\n",
      "aft 1\n",
      "aimé 9\n",
      "axis 2\n",
      "acte 3\n",
      "artificially 2\n",
      "accorder 1\n",
      "administrator 5\n",
      "algebra 1\n",
      "arrivez 1\n",
      "alert 16\n",
      "anatomy 1\n",
      "afterlife 1\n",
      "abuse 26\n",
      "adventure 4\n",
      "amènent 2\n",
      "accountant 10\n",
      "authentic 3\n",
      "appoint 3\n",
      "agirons 1\n",
      "Appelez 4\n",
      "amid 52\n",
      "allant 1\n",
      "attachement 1\n",
      "arduous 1\n",
      "An 143\n",
      "aiguille 2\n",
      "appertain 1\n",
      "Abramovna 1\n",
      "Aussitôt 1\n",
      "alack 7\n",
      "assistance 11\n",
      "Allons 30\n",
      "admirer 2\n",
      "advised 24\n",
      "Après 1\n",
      "aghast 4\n",
      "appeals 1\n",
      "associée 1\n",
      "activates 5\n",
      "agrees 5\n",
      "adorning 1\n",
      "awakens 1\n",
      "Archduke 12\n",
      "apex 1\n",
      "Alexeevna 2\n",
      "advancement 2\n",
      "actually 17\n",
      "armaments 2\n",
      "Adjutant 6\n",
      "aliens 4\n",
      "antagonism 1\n",
      "auto 1\n",
      "annoncer 1\n",
      "anonymous 3\n",
      "Applies 1\n",
      "Aroint 1\n",
      "Astraea 1\n",
      "awaken 5\n",
      "avenues 2\n",
      "allument 1\n",
      "annihilate 3\n",
      "ARTOO 1\n",
      "abîmer 1\n",
      "atteinte 1\n",
      "Availing 1\n",
      "arrows 3\n",
      "Allez 14\n",
      "anxious 40\n",
      "apparences 4\n",
      "appetizing 2\n",
      "Alike 1\n",
      "Affliction 1\n",
      "affermis 1\n",
      "awash 1\n",
      "accuses 3\n",
      "aliment 1\n",
      "absolutely 23\n",
      "addeth 1\n",
      "abnormal 1\n",
      "Anyway 1\n",
      "apparence 2\n",
      "Annette 12\n",
      "arc 5\n",
      "asleep 96\n",
      "apologies 1\n",
      "affectation 12\n",
      "avoided 23\n",
      "angelical 1\n",
      "abide 11\n",
      "Authorizing 1\n",
      "Alas 44\n",
      "affecting 4\n",
      "ax 9\n",
      "Along 5\n",
      "annoncent 1\n",
      "affirming 1\n",
      "abhorring 1\n",
      "appertaining 1\n",
      "Amyntas 1\n",
      "august 4\n",
      "AGRIPPA 41\n",
      "apportioned 2\n",
      "Anselme 1\n",
      "anglaise 4\n",
      "advantages 15\n",
      "answers 22\n",
      "abate 3\n",
      "agnise 1\n",
      "approval 24\n",
      "Angel 3\n",
      "ANYTHING 2\n",
      "Andre 2\n",
      "appartenait 1\n",
      "abolition 1\n",
      "accouterments 1\n",
      "argue 8\n",
      "abbreviations 1\n",
      "attendent 2\n",
      "appeler 6\n",
      "AEneas 1\n",
      "ANNIE 2\n",
      "adroit 7\n",
      "Angry 3\n",
      "anti 1\n",
      "apologize 11\n",
      "accumulated 1\n",
      "alluringly 1\n",
      "aback 3\n",
      "Audio 1\n",
      "argued 6\n",
      "altitude 2\n",
      "après 21\n",
      "agrement 1\n",
      "ALLSOPP 1\n",
      "amenities 1\n",
      "arme 2\n",
      "Albanians 1\n",
      "annals 1\n",
      "afford 5\n",
      "acteurs 1\n",
      "absorbe 1\n",
      "admonish 2\n",
      "apprehensions 1\n",
      "Allied 1\n",
      "ancienne 3\n",
      "assail 2\n",
      "albums 4\n",
      "agréable 3\n",
      "abusé 1\n",
      "And 2616\n",
      "ALL 14\n",
      "Apraksins 2\n",
      "applying 6\n",
      "ace 1\n",
      "affirm 1\n",
      "ait 20\n",
      "aisé 1\n",
      "alehouse 1\n",
      "affaiblissent 1\n",
      "aloofness 2\n",
      "almonds 1\n",
      "accustomed 44\n",
      "assays 1\n",
      "Aide 3\n",
      "attaches 3\n",
      "amble 2\n",
      "aimerais 3\n",
      "addressing 72\n",
      "Answering 1\n",
      "am 935\n",
      "absurdes 1\n",
      "avide 1\n",
      "assumptions 2\n",
      "apoplectic 1\n",
      "attain 21\n",
      "ans 4\n",
      "adoucirait 1\n",
      "amused 26\n",
      "Admiral 32\n",
      "accoucheur 1\n",
      "accordé 1\n",
      "ancestors 9\n",
      "anymore 2\n",
      "assails 1\n",
      "alors 17\n",
      "answering 31\n",
      "aside 76\n",
      "arrangement 8\n",
      "amazing 5\n",
      "absorbed 32\n",
      "ambitious 13\n",
      "agitated 48\n",
      "afflicts 1\n",
      "Alarmed 2\n",
      "Azov 1\n",
      "appliqué 1\n",
      "Absent 1\n",
      "animent 1\n",
      "arrêt 1\n",
      "averted 1\n",
      "alarum 4\n",
      "Admit 4\n",
      "arbiter 1\n",
      "adder 2\n",
      "authorities 22\n",
      "accepting 5\n",
      "aware 27\n",
      "approved 16\n",
      "ambiguous 1\n",
      "amené 4\n",
      "arson 2\n",
      "animatedly 1\n",
      "admiring 6\n",
      "astonishing 2\n",
      "ascertained 1\n",
      "ambulance 5\n",
      "appellent 1\n",
      "amazed 29\n",
      "Alliance 15\n",
      "appease 3\n",
      "aggraver 1\n",
      "appauvrit 1\n",
      "accurate 3\n",
      "actress 4\n",
      "aroma 1\n",
      "avalanche 2\n",
      "aider 1\n",
      "avare 1\n",
      "ARNELL 1\n",
      "afflicted 2\n",
      "alimony 1\n",
      "acquis 1\n",
      "airspeeder 1\n",
      "ariseth 1\n",
      "Assist 1\n",
      "assertion 1\n",
      "astonish 5\n",
      "alight 3\n",
      "Akinfi 1\n",
      "allaiter 1\n",
      "Amalek 2\n",
      "Affront 1\n",
      "advances 4\n",
      "ame 3\n",
      "accrue 1\n",
      "assurée 1\n",
      "afore 2\n",
      "abduction 3\n",
      "amerce 1\n",
      "aweary 2\n",
      "ascended 12\n",
      "arrange 29\n",
      "Alexandria 19\n",
      "ASCHKYNAZO 1\n",
      "always 474\n",
      "avons 13\n",
      "admitted 30\n",
      "ashes 6\n",
      "ascertain 5\n",
      "accuracy 5\n",
      "attracting 7\n",
      "amoureux 3\n",
      "Arrêtez 3\n",
      "Awkward 1\n",
      "Aren 7\n",
      "almond 1\n",
      "Arms 1\n",
      "Alley 2\n",
      "animate 2\n",
      "appreciating 1\n",
      "Augustin 2\n",
      "Apraksina 5\n",
      "affects 1\n",
      "avouch 2\n",
      "accuser 2\n",
      "Amazon 1\n",
      "Abuse 1\n",
      "Arakcheev 43\n",
      "above 211\n",
      "Ambition 1\n",
      "Avaunt 3\n",
      "avoir 32\n",
      "alertness 1\n",
      "attouchement 1\n",
      "AROUND 28\n",
      "adoucir 1\n",
      "ambled 1\n",
      "Amuse 1\n",
      "ardeur 4\n",
      "antechamber 5\n",
      "Achtung 1\n",
      "Aaargh 1\n",
      "abstemious 1\n",
      "affording 3\n",
      "affronts 2\n",
      "allez 9\n",
      "adventurer 3\n",
      "alligator 1\n",
      "avaricious 1\n",
      "affamés 1\n",
      "admis 1\n",
      "advint 1\n",
      "Arabie 1\n",
      "aridité 1\n",
      "astute 1\n",
      "Absolute 2\n",
      "apple 9\n",
      "adroitness 3\n",
      "antidote 1\n",
      "aimez 2\n",
      "Asia 2\n",
      "Arcola 3\n",
      "AFTERNOON 2\n",
      "arsenic 1\n",
      "actes 2\n",
      "ape 5\n",
      "Anthony 1\n",
      "acquaint 1\n",
      "avait 52\n",
      "asses 1\n",
      "accorde 1\n",
      "abroad 43\n",
      "avez 48\n",
      "almighty 2\n",
      "apprivoise 1\n",
      "autrement 6\n",
      "airy 6\n",
      "abusing 3\n",
      "agility 9\n",
      "argument 27\n",
      "ANTONY 308\n",
      "awake 35\n",
      "antic 4\n",
      "allopaths 1\n",
      "abstained 2\n",
      "abandoned 55\n",
      "affray 1\n",
      "avoiding 18\n",
      "aggressor 1\n",
      "Absolutely 1\n",
      "appositely 1\n",
      "anew 7\n",
      "abuser 1\n",
      "apiece 1\n",
      "accessed 2\n",
      "Angelo 2\n",
      "Angle 2\n",
      "angel 33\n",
      "Anton 3\n",
      "abstract 11\n",
      "Athens 4\n",
      "auraient 3\n",
      "alike 31\n",
      "aveuglement 1\n",
      "abhors 1\n",
      "Associate 1\n",
      "Arguing 1\n",
      "agony 9\n",
      "Adonai 1\n",
      "Accountants 2\n",
      "acquaintances 39\n",
      "attachment 1\n",
      "angered 4\n",
      "ambuscade 1\n",
      "aimera 1\n",
      "AUER 1\n",
      "absolver 1\n",
      "accidental 4\n",
      "admonishingly 1\n",
      "AK 2\n",
      "arduously 1\n",
      "affranchira 1\n",
      "assenting 1\n",
      "action 247\n",
      "awl 3\n",
      "afterglow 1\n",
      "arose 22\n",
      "amazeth 1\n",
      "amiably 8\n",
      "apparell 1\n",
      "approving 3\n",
      "awkwardly 14\n",
      "alertly 1\n",
      "arising 5\n",
      "adopt 8\n",
      "Ascend 1\n",
      "accidentelle 1\n",
      "asked 675\n",
      "acquitted 1\n",
      "accounted 4\n",
      "asserts 1\n",
      "Artists 3\n",
      "adjutant 160\n",
      "Armed 2\n",
      "authoritatively 1\n",
      "aidé 1\n",
      "Anglais 3\n",
      "adjustments 5\n",
      "arranging 18\n",
      "aides 19\n",
      "alternative 3\n",
      "Alert 3\n",
      "amusingly 1\n",
      "APOSTOCOS 1\n",
      "apartment 5\n",
      "abbey 1\n",
      "arrogance 1\n",
      "alchemy 3\n",
      "avis 7\n",
      "accentuating 2\n",
      "ado 1\n",
      "Apsherons 4\n",
      "actif 1\n",
      "amants 6\n",
      "adapt 1\n",
      "ami 20\n",
      "appointment 20\n",
      "ample 2\n",
      "auspicious 1\n",
      "Aboard 1\n",
      "altogether 14\n",
      "ayant 2\n",
      "association 1\n",
      "Ambassador 3\n",
      "annihilated 1\n",
      "assignation 1\n",
      "abandonnent 1\n",
      "altering 5\n",
      "Auerstadt 4\n",
      "accordance 9\n",
      "audience 32\n",
      "abodes 1\n",
      "apparatuses 1\n",
      "accosté 1\n",
      "ANNOUNCER 3\n",
      "ACT 15\n",
      "Aside 60\n",
      "authority 35\n",
      "approximation 2\n",
      "accused 11\n",
      "are 2166\n",
      "ascent 2\n",
      "altercation 1\n",
      "Abbe 3\n",
      "acquaintance 46\n",
      "acknowledgment 3\n",
      "aboveboard 1\n",
      "appreciative 2\n",
      "Adder 1\n",
      "allowing 6\n",
      "archduke 2\n",
      "assured 33\n",
      "apparition 5\n",
      "Ante 1\n",
      "aventurière 1\n",
      "active 27\n",
      "ache 7\n",
      "accompagner 1\n",
      "AGREE 4\n",
      "Aubert 1\n",
      "adept 2\n",
      "attainted 1\n",
      "according 70\n",
      "Augezd 1\n",
      "ANTHONY 3\n",
      "arrière 1\n",
      "anecdote 4\n",
      "allowance 8\n",
      "argumentative 1\n",
      "attaint 1\n",
      "angrily 86\n",
      "arrangerai 1\n",
      "antiquity 3\n",
      "ask 225\n",
      "atténuer 1\n",
      "affliction 5\n",
      "ailing 1\n",
      "additions 3\n",
      "armed 32\n",
      "Aeneas 2\n",
      "additional 9\n",
      "amazingly 1\n",
      "approximately 4\n",
      "accommodation 3\n",
      "asteroid 33\n",
      "attends 4\n",
      "achievement 6\n",
      "aperçoit 2\n",
      "adroitly 2\n",
      "artless 3\n",
      "attendants 3\n",
      "aies 1\n",
      "administrators 1\n",
      "Alarms 5\n",
      "amant 4\n",
      "adoucisse 1\n",
      "affaires 15\n",
      "Aghast 1\n",
      "apples 2\n",
      "abuses 5\n",
      "aie 9\n",
      "Antilles 3\n",
      "anyways 2\n",
      "accablante 1\n",
      "Austerlitz 51\n",
      "articulating 4\n",
      "actuated 2\n",
      "alley 1\n",
      "adventures 7\n",
      "antagonistic 1\n",
      "Animation 4\n",
      "assay 7\n",
      "Alluvial 1\n",
      "affairs 133\n",
      "arrêtez 4\n",
      "aisément 3\n",
      "alphabet 2\n",
      "Attention 2\n",
      "able 126\n",
      "agonizing 2\n",
      "amoureuse 2\n",
      "afwaid 1\n",
      "astride 1\n",
      "accompagne 2\n",
      "Adjutants 1\n",
      "abstraction 6\n",
      "abdicate 1\n",
      "attend 33\n",
      "absolved 2\n",
      "ardour 1\n",
      "Alack 15\n",
      "aillent 1\n",
      "abdomens 2\n",
      "asylums 3\n",
      "ARMSTRONG 2\n",
      "assume 31\n",
      "average 1\n",
      "analysis 4\n",
      "As 636\n",
      "accidents 11\n",
      "aimai 2\n",
      "attractively 1\n",
      "availing 7\n",
      "accumulate 2\n",
      "Assuming 3\n",
      "afin 2\n",
      "aggressive 2\n",
      "aggression 4\n",
      "aiguillon 1\n",
      "Artoo 368\n",
      "alluded 5\n",
      "alternate 4\n",
      "attraction 10\n",
      "Awake 5\n",
      "attractive 20\n",
      "Afore 1\n",
      "ANGELINETTA 1\n",
      "aux 43\n",
      "aglow 3\n",
      "alternating 1\n",
      "apron 12\n",
      "auguring 1\n",
      "Alpatych 137\n",
      "adversaries 2\n",
      "ardor 5\n",
      "allons 13\n",
      "agitating 6\n",
      "Antonov 1\n",
      "awaked 4\n",
      "Aie 1\n",
      "Animator 1\n",
      "abyss 11\n",
      "ALEX 2\n",
      "anxiously 19\n",
      "advent 1\n",
      "accidentally 6\n",
      "Authority 1\n",
      "arborer 1\n",
      "ATTACK 1\n",
      "apathetic 1\n",
      "accompanied 47\n",
      "allegories 1\n",
      "activate 1\n",
      "acquired 12\n",
      "ancients 16\n",
      "assassiner 2\n",
      "Avez 4\n",
      "assumption 7\n",
      "appurtenance 1\n",
      "amicable 1\n",
      "assigns 2\n",
      "accomplish 4\n",
      "Artist 8\n",
      "antipathetic 1\n",
      "asking 113\n",
      "accès 3\n",
      "avenge 3\n",
      "Arriving 2\n",
      "Awkwardly 1\n",
      "assurance 29\n",
      "advantageously 3\n",
      "abridgement 1\n",
      "amen 8\n",
      "AM 1\n",
      "Anybody 1\n",
      "awards 4\n",
      "Amour 1\n",
      "aimerez 1\n",
      "Annoyed 1\n",
      "Anisya 16\n",
      "auction 2\n",
      "abstinence 1\n",
      "Angrily 1\n",
      "agree 58\n",
      "accomplisse 1\n",
      "antecedents 1\n",
      "assassin 1\n",
      "approve 22\n",
      "amber 2\n",
      "AILSA 1\n",
      "addresses 3\n",
      "Andwew 1\n",
      "addressed 61\n",
      "atroce 1\n",
      "artifice 1\n",
      "allais 2\n",
      "array 15\n",
      "armee 3\n",
      "Ami 1\n",
      "adroite 1\n",
      "ancient 45\n",
      "Assembly 1\n",
      "asks 18\n",
      "AREA 19\n",
      "aye 3\n",
      "Adallas 1\n",
      "accent 14\n",
      "accordingly 6\n",
      "applique 1\n",
      "aristocracy 3\n",
      "aveu 1\n",
      "Alps 1\n",
      "Although 15\n",
      "alcove 9\n",
      "adjusting 6\n",
      "antipathy 7\n",
      "appreciation 4\n",
      "affirmative 2\n",
      "appointed 51\n",
      "aid 30\n",
      "Anyone 2\n",
      "animals 18\n",
      "apprehension 5\n",
      "Africa 8\n",
      "appealing 1\n",
      "aversion 2\n",
      "assassiné 2\n",
      "advisable 1\n",
      "appraisal 1\n",
      "Any 11\n",
      "and 26617\n",
      "antre 1\n",
      "appris 2\n",
      "anteater 1\n",
      "Alors 3\n",
      "awaking 1\n",
      "an 2061\n",
      "Almighty 6\n",
      "alerted 2\n",
      "Above 13\n",
      "allowed 46\n",
      "amène 2\n",
      "Angels 2\n",
      "Avenger 7\n",
      "amours 4\n",
      "afraid 172\n",
      "allege 1\n",
      "adoring 2\n",
      "apprise 1\n",
      "admittance 1\n",
      "adverse 1\n",
      "Andrews 1\n",
      "ANOTHER 5\n",
      "Ask 10\n",
      "Antoniad 1\n",
      "adve 1\n",
      "anoint 1\n",
      "annex 3\n",
      "astounding 2\n",
      "ANDREW 1\n",
      "abandonna 1\n",
      "Anywhere 2\n",
      "apprenticeship 2\n",
      "abandonnerai 2\n",
      "advocated 1\n",
      "Asteroids 6\n",
      "attaining 4\n",
      "avances 1\n",
      "accusing 2\n",
      "arbitrarily 5\n",
      "assembling 2\n",
      "Aye 1\n",
      "anon 17\n",
      "Africans 1\n",
      "aveux 1\n",
      "argues 3\n",
      "Anne 1\n",
      "abundant 3\n",
      "avenue 14\n",
      "abandonment 14\n",
      "admirably 2\n",
      "abandoning 25\n",
      "acknowledge 9\n",
      "anointed 2\n",
      "admission 3\n",
      "amounted 4\n",
      "abhorred 3\n",
      "atrocities 1\n",
      "aimait 4\n",
      "AIR 2\n",
      "adjoin 1\n",
      "armchair 32\n",
      "adviser 1\n",
      "artilleryman 6\n",
      "affected 19\n",
      "advantage 46\n",
      "appal 2\n",
      "Amid 8\n",
      "aviez 2\n",
      "advocate 1\n",
      "Alexeevich 41\n",
      "Adam 6\n",
      "actuelles 1\n",
      "arriving 14\n",
      "awaiting 46\n",
      "affections 12\n",
      "afire 1\n",
      "avoidance 2\n",
      "aveuglé 1\n",
      "annexment 1\n",
      "Arise 6\n",
      "addicted 1\n",
      "ALEXANDER 1\n",
      "acknowledging 6\n",
      "artificiality 3\n",
      "Acquaint 2\n",
      "apartments 13\n",
      "alterations 1\n",
      "Answer 4\n",
      "approof 1\n",
      "agir 1\n",
      "affable 4\n",
      "agriculture 4\n",
      "amusement 7\n",
      "attempting 3\n",
      "armor 4\n",
      "ASSISTANT 1\n",
      "abounding 1\n",
      "Askance 1\n",
      "appeal 17\n",
      "armpit 1\n",
      "ass 11\n",
      "Anthropophagi 1\n",
      "appetites 3\n",
      "ascribed 1\n",
      "aille 1\n",
      "atoms 3\n",
      "attending 19\n",
      "apporté 1\n",
      "although 15\n",
      "assemble 4\n",
      "Assuredly 1\n",
      "about 1214\n",
      "Already 7\n",
      "allies 11\n",
      "anything 314\n",
      "aucune 4\n",
      "anomalous 1\n",
      "abusée 1\n",
      "Antiquity 1\n",
      "adored 14\n",
      "approche 2\n",
      "act 106\n",
      "arrogantly 2\n",
      "ALEC 4\n",
      "aspics 1\n",
      "Angelica 1\n",
      "affectionately 10\n",
      "admired 17\n",
      "auntie 1\n",
      "AMAND 1\n",
      "Away 18\n",
      "Abbey 1\n",
      "alleyway 4\n",
      "Acting 1\n",
      "auriez 1\n",
      "ante 1\n",
      "Aux 2\n",
      "addition 17\n",
      "argumentatively 1\n",
      "abdique 1\n",
      "abstaining 2\n",
      "afterward 1\n",
      "ANDERSON 3\n",
      "asset 1\n",
      "acre 1\n",
      "adopter 1\n",
      "angrier 3\n",
      "Avance 1\n",
      "authorized 2\n",
      "alienated 1\n",
      "animaux 1\n",
      "ainsi 47\n",
      "Aucun 1\n",
      "acquire 4\n",
      "antennae 2\n",
      "accompaniment 2\n",
      "ashamed 60\n",
      "atteindre 1\n",
      "avaunt 1\n",
      "appreciated 13\n",
      "arranged 49\n",
      "Ansicht 1\n",
      "affligée 1\n",
      "architect 13\n",
      "applies 2\n",
      "abandon 28\n",
      "absently 6\n",
      "arrive 36\n",
      "Affairs 2\n",
      "absorb 1\n",
      "ab 1\n",
      "accusais 1\n",
      "Additional 6\n",
      "absolument 3\n",
      "Armenians 1\n",
      "AGREEMENT 2\n",
      "atone 7\n",
      "advising 3\n",
      "abandonnée 2\n",
      "aches 5\n",
      "Avouez 1\n",
      "ambush 7\n",
      "absences 1\n",
      "author 12\n",
      "accorderais 1\n",
      "allumée 1\n",
      "attributing 2\n",
      "animation 29\n",
      "ancêtres 1\n",
      "Adventure 1\n",
      "ashore 2\n",
      "attempted 6\n",
      "announcements 2\n",
      "ancle 1\n",
      "Articles 1\n",
      "arriver 3\n",
      "albâtre 1\n",
      "aimerai 3\n",
      "AL 1\n",
      "auberge 1\n",
      "apprends 1\n",
      "Avancez 1\n",
      "animosity 3\n",
      "Antony 202\n",
      "a 14250\n",
      "Arabchik 1\n",
      "All 448\n",
      "aimless 4\n",
      "agi 1\n",
      "Almost 12\n",
      "amount 7\n",
      "ACKBAR 15\n",
      "amazedly 1\n",
      "Aww 1\n",
      "agencies 1\n",
      "Au 17\n",
      "Actium 3\n",
      "axe 5\n",
      "azur 1\n",
      "Alphonse 1\n",
      "avancer 2\n",
      "aided 2\n",
      "affair 83\n",
      "arming 2\n",
      "Appear 3\n",
      "abondamment 1\n",
      "asteroids 13\n",
      "aperçu 1\n",
      "ajouté 2\n",
      "Andrew 1142\n",
      "aspect 11\n",
      "amplified 1\n",
      "assembled 37\n",
      "acquitterai 1\n",
      "Arbat 8\n",
      "agile 4\n",
      "acres 4\n",
      "Animatronics 1\n",
      "abused 18\n",
      "arbiters 1\n",
      "abashed 12\n",
      "Anything 1\n",
      "AFFONSO 1\n",
      "avouer 2\n",
      "Altogether 1\n",
      "admitting 15\n",
      "anteroom 37\n",
      "abundance 8\n",
      "abound 2\n",
      "AGOSTINO 1\n",
      "arrache 1\n",
      "adventurous 1\n",
      "aggressively 1\n",
      "America 1\n",
      "affianced 6\n",
      "Aliment 1\n",
      "almanacs 1\n",
      "arrested 19\n",
      "alteration 4\n",
      "accompanies 2\n",
      "arrivée 3\n",
      "activity 120\n",
      "Anger 4\n",
      "analyzing 2\n",
      "abolishing 1\n",
      "ASCII 4\n",
      "adulterous 1\n",
      "approaching 81\n",
      "aventure 5\n",
      "abominably 2\n",
      "armies 47\n",
      "alternated 1\n",
      "arouses 1\n",
      "Antonius 10\n",
      "actively 3\n",
      "Authorized 1\n",
      "awry 6\n",
      "Auprès 1\n",
      "adherents 4\n",
      "autrui 2\n",
      "arbre 1\n",
      "Antonovna 4\n",
      "assisted 1\n",
      "abnegation 2\n",
      "adhere 2\n",
      "adversities 1\n",
      "AUNT 6\n",
      "allotted 4\n",
      "assistants 4\n",
      "ammunition 14\n",
      "announced 32\n",
      "aimât 1\n",
      "attended 22\n",
      "ability 7\n",
      "Armfeldts 1\n",
      "arbitrary 6\n",
      "agreement 53\n",
      "adage 1\n",
      "agrément 1\n",
      "astir 2\n",
      "ash 1\n",
      "attendez 1\n",
      "ange 2\n",
      "acquiring 2\n",
      "appartenir 1\n",
      "ayons 1\n",
      "Anchises 1\n",
      "appendages 1\n",
      "abandonner 1\n",
      "alliee 1\n",
      "advisers 8\n",
      "afar 6\n",
      "abordé 1\n",
      "altérer 2\n",
      "aimes 1\n",
      "autant 11\n",
      "attains 1\n",
      "accusation 1\n",
      "authoritative 1\n",
      "Attends 1\n",
      "another 699\n",
      "amoncelés 1\n",
      "Adorable 3\n",
      "Apollon 1\n",
      "attraperai 1\n",
      "arts 3\n",
      "alarmed 13\n",
      "arbitrariness 1\n",
      "Attend 2\n",
      "acceptance 3\n",
      "abhorre 1\n",
      "approximate 2\n",
      "awarded 5\n",
      "attach 8\n",
      "Antick 1\n",
      "allows 4\n",
      "asiatique 2\n",
      "affectionate 20\n",
      "Assume 1\n",
      "altar 6\n",
      "aborder 1\n",
      "Activate 1\n",
      "Achilles 6\n",
      "astounded 3\n",
      "annoy 2\n",
      "Admired 1\n",
      "appelée 1\n",
      "affronted 1\n",
      "accablantes 1\n",
      "amend 2\n",
      "Applauding 1\n",
      "ADVENTURER 1\n",
      "ais 1\n",
      "antechambers 1\n",
      "arranger 1\n",
      "arouse 10\n",
      "Apropos 1\n",
      "aime 29\n",
      "abatements 1\n",
      "almshouse 1\n",
      "archway 1\n",
      "affection 32\n",
      "Archives 1\n",
      "awestruck 1\n",
      "agonized 2\n",
      "armored 15\n",
      "aqua 2\n",
      "amiabilities 2\n",
      "Amidst 1\n",
      "anathematized 1\n",
      "awakening 10\n",
      "amollissaient 1\n",
      "arise 14\n",
      "amour 55\n",
      "apoplex 1\n",
      "aptitude 3\n",
      "agreeing 2\n",
      "approuve 1\n",
      "Austria 32\n",
      "aimée 3\n",
      "approvingly 6\n",
      "allaient 1\n",
      "attachent 1\n",
      "Authorities 1\n",
      "alabaster 1\n",
      "allot 1\n",
      "anticipatest 1\n",
      "arrives 8\n",
      "applaud 4\n",
      "after 734\n",
      "achevé 1\n",
      "applications 1\n",
      "assault 12\n",
      "accomplir 3\n",
      "Advertising 1\n",
      "attent 1\n",
      "Act 28\n",
      "airless 1\n",
      "alpha 1\n",
      "astronomy 9\n",
      "assassins 1\n",
      "agitent 1\n",
      "askance 3\n",
      "avant 13\n",
      "Amstetten 1\n",
      "advocates 3\n",
      "Asleep 1\n",
      "At 383\n",
      "atrocity 1\n",
      "Animated 1\n",
      "assaut 1\n",
      "abilities 4\n",
      "apaiser 1\n",
      "Austro 1\n",
      "actions 82\n",
      "aimer 6\n",
      "augure 1\n",
      "arithmetician 1\n",
      "Advise 1\n",
      "announcement 8\n",
      "adoration 5\n",
      "Ainsi 10\n",
      "Advantage 1\n",
      "abominations 1\n",
      "Acquire 1\n",
      "abjure 1\n",
      "ARTHUR 2\n",
      "augurers 3\n",
      "Anyhow 2\n",
      "admirable 13\n",
      "ally 12\n",
      "ACADÉMIQUE 1\n",
      "accompli 1\n",
      "adultère 1\n",
      "aspects 1\n",
      "abandons 1\n",
      "audibly 3\n",
      "alighting 2\n",
      "ague 3\n",
      "attacking 25\n",
      "ALBERT 1\n",
      "Anon 11\n",
      "adhered 1\n",
      "apart 65\n",
      "articles 8\n",
      "abhorrence 1\n",
      "acme 2\n",
      "assemblies 1\n",
      "ANAKIN 2\n",
      "autrefois 1\n",
      "answerest 1\n",
      "accounts 22\n",
      "appelais 1\n",
      "accumulation 1\n",
      "Actually 2\n",
      "arrivals 7\n",
      "apothecary 3\n",
      "Ahhh 3\n",
      "ALLEYWAY 2\n",
      "Aw 1\n",
      "aurais 10\n",
      "allusions 4\n",
      "Asiatic 3\n",
      "appeased 3\n",
      "agissent 1\n",
      "alarming 2\n",
      "apologetically 2\n",
      "attacker 3\n",
      "ailes 2\n",
      "access 25\n",
      "Andrusha 6\n",
      "attire 7\n",
      "agisse 1\n",
      "attentions 6\n",
      "archduchess 2\n",
      "awistocwacy 1\n",
      "attention 131\n",
      "avertir 1\n",
      "aloft 4\n",
      "actresses 3\n",
      "approchaient 1\n",
      "Armature 1\n",
      "alters 2\n",
      "assiduously 1\n",
      "assailable 1\n",
      "accablée 1\n",
      "aisle 1\n",
      "ate 17\n",
      "assiduity 1\n",
      "adores 2\n",
      "aveugle 2\n",
      "Andreevich 8\n",
      "ANN 1\n",
      "Aahhh 1\n",
      "assomment 1\n",
      "assure 21\n",
      "allied 2\n",
      "alliance 17\n",
      "afterwards 41\n",
      "attache 1\n",
      "arsenal 4\n",
      "arras 5\n",
      "agents 3\n",
      "actuels 1\n",
      "afoot 2\n",
      "aller 11\n",
      "ABRAHAM 6\n",
      "August 43\n",
      "army 672\n",
      "Academy 7\n",
      "attack 147\n",
      "AUGUSTINS 1\n",
      "assuring 7\n",
      "aimable 2\n",
      "ankle 3\n",
      "agreeable 27\n",
      "assumes 4\n",
      "adapting 3\n",
      "amongst 2\n",
      "aimante 1\n",
      "amertume 2\n",
      "account 81\n",
      "ALF 1\n",
      "amene 1\n",
      "accomplishment 2\n",
      "appearance 46\n",
      "air 222\n",
      "affronter 1\n",
      "Adele 1\n",
      "atheist 1\n",
      "Archduchy 1\n",
      "audiences 1\n",
      "Apraksin 1\n",
      "adorn 2\n",
      "abode 2\n",
      "abord 7\n",
      "aerial 4\n",
      "aujourd 5\n",
      "arching 3\n",
      "appellerait 1\n",
      "ant 1\n",
      "as 5151\n",
      "associating 2\n",
      "ahahah 1\n",
      "anger 69\n",
      "any 694\n",
      "again 874\n",
      "Around 10\n",
      "antenna 3\n",
      "auger 1\n",
      "addle 1\n",
      "answerable 1\n",
      "abjurer 1\n",
      "arithmetic 3\n",
      "About 27\n",
      "altered 20\n",
      "Apothecary 5\n",
      "arguing 8\n",
      "attachée 1\n",
      "April 12\n",
      "asile 2\n",
      "Assays 1\n",
      "attachant 1\n",
      "arch 5\n",
      "addiction 1\n",
      "athwart 1\n",
      "attempt 29\n",
      "advantageous 15\n",
      "ailleurs 6\n",
      "acceptés 1\n",
      "Ah 247\n",
      "arbitrate 1\n",
      "avoue 7\n",
      "admonishing 1\n",
      "Accelerate 2\n",
      "allow 58\n",
      "assouvir 2\n",
      "abhor 4\n",
      "all 3390\n",
      "Angleterre 3\n",
      "AVENGER 3\n",
      "agreed 36\n",
      "augury 1\n",
      "atmosphere 19\n",
      "assigned 13\n",
      "astonished 18\n",
      "attitude 37\n",
      "Arabian 2\n",
      "arrears 1\n",
      "avouches 1\n",
      "adjunct 2\n",
      "appraising 1\n",
      "Anthropology 1\n",
      "adoption 1\n",
      "Anonymous 2\n",
      "armour 11\n",
      "amusing 21\n",
      "activities 12\n",
      "animal 38\n",
      "arbours 1\n",
      "appraised 1\n",
      "assistant 26\n",
      "adventurers 2\n",
      "arcing 2\n",
      "AGGIE 1\n",
      "awaited 35\n",
      "Almain 1\n",
      "affability 2\n",
      "anticipates 1\n",
      "aiming 7\n",
      "accourant 1\n",
      "Affection 2\n",
      "afeard 7\n",
      "ASTRO 1\n",
      "artillerymen 8\n",
      "ancien 1\n",
      "aggrieved 1\n",
      "appelait 2\n",
      "assuré 1\n",
      "airiness 1\n",
      "Allemand 2\n",
      "achieved 7\n",
      "Addition 1\n",
      "Atlas 1\n",
      "adulation 2\n",
      "ABC 1\n",
      "accelerate 2\n",
      "arrivées 1\n",
      "acid 1\n",
      "aurait 4\n",
      "Assessor 1\n",
      "akimbo 4\n",
      "advocation 1\n",
      "Advances 2\n",
      "Anferovs 2\n",
      "axes 2\n",
      "artificial 16\n",
      "accurst 1\n",
      "allegory 1\n",
      "abrupt 6\n",
      "Attendant 13\n",
      "approach 57\n",
      "Appears 1\n",
      "asp 3\n",
      "assiste 1\n",
      "Against 31\n",
      "AYERS 1\n",
      "attainment 7\n",
      "amazement 16\n",
      "Among 28\n",
      "abasement 1\n",
      "Allusion 1\n",
      "adorer 4\n",
      "astwide 2\n",
      "alte 1\n",
      "annual 1\n",
      "Antilies 1\n",
      "Apt 1\n",
      "Arm 4\n",
      "acclamations 2\n",
      "audacity 4\n",
      "afternoon 19\n",
      "adapted 2\n",
      "Approchez 1\n",
      "autour 3\n",
      "Alarums 7\n",
      "ants 5\n",
      "Aleppo 2\n",
      "abed 2\n",
      "adopted 17\n",
      "agwee 1\n",
      "approchez 1\n",
      "Alexandrian 2\n",
      "ANNE 1\n",
      "appelle 4\n",
      "avec 140\n",
      "apparently 40\n",
      "Alarum 12\n",
      "allured 1\n",
      "Ashamed 1\n",
      "Abroad 2\n",
      "armistice 4\n",
      "Alexander 145\n",
      "abridged 1\n",
      "advance 72\n",
      "auront 1\n",
      "adresse 2\n",
      "avions 2\n",
      "assert 2\n",
      "auparavant 2\n",
      "Action 2\n",
      "attentive 17\n",
      "applied 10\n",
      "absent 45\n",
      "admiration 14\n",
      "apology 2\n",
      "Aniska 3\n",
      "arrant 2\n",
      "aim 123\n",
      "alluding 7\n",
      "ASTEROID 20\n",
      "accidently 1\n",
      "annihilating 1\n",
      "année 6\n",
      "agate 3\n",
      "avouera 1\n",
      "associated 23\n",
      "assent 3\n",
      "amphitheater 2\n",
      "ar 1\n",
      "ablaze 3\n",
      "artificieuse 1\n",
      "Awfully 3\n",
      "Administrative 1\n",
      "admirables 1\n",
      "assurances 2\n",
      "acts 14\n",
      "assassinats 1\n",
      "aromatic 2\n",
      "anarchy 1\n",
      "ambuscadoes 1\n",
      "achetés 1\n",
      "arguments 22\n",
      "Accuse 1\n",
      "armé 2\n",
      "answer 211\n",
      "abysm 2\n",
      "A 1089\n",
      "Ate 2\n",
      "amiable 15\n",
      "approves 1\n",
      "await 16\n",
      "Arranger 1\n",
      "abandonne 1\n",
      "avance 6\n",
      "auxiliary 3\n",
      "aise 6\n",
      "affirmer 1\n",
      "After 227\n",
      "ARTEMIDORUS 7\n",
      "assign 2\n",
      "amputated 3\n",
      "aunt 38\n",
      "Amiss 1\n",
      "antres 1\n",
      "amis 11\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "\n",
    "# Compile a regular expression that matches non-alphanumerics\n",
    "pattern = re.compile('[\\W_]+', re.UNICODE)\n",
    "\n",
    "# Replace all non-alphanumerics with a space, then split into words\n",
    "words = textFile.map(lambda line: pattern.sub(' ',line)).flatMap(lambda line: line.split())\n",
    "\n",
    "#first_letters = set(['H','h','Q','q','s','S'])\n",
    "first_letters = set(['A','a'])\n",
    "# Count words that start with H\n",
    "words_subset = words.filter(lambda x: x[0] in first_letters)\n",
    "counts = words_subset.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "res = counts.collect()\n",
    "for r,c in res:\n",
    "    print r,c"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Counting Letter bigrams\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import re\n",
    "\n",
    "#textFile = sc.textFile(\"notes/data/books-eng/hamlet.txt\")\n",
    "textFile = sc.textFile(\"data/books-eng\")\n",
    "\n",
    "# Compile a regular expression that matches non-alphanumerics\n",
    "pattern = re.compile(u'[\\W0-9_]+', re.UNICODE)\n",
    "\n",
    "# Replace all non-alphanumerics with a space, then split into words\n",
    "words = textFile.map(lambda line: pattern.sub(' ',line)).flatMap(lambda line: line.split())\n",
    "# Convert to lower case\n",
    "words = words.map(lambda w: w.lower())\n",
    "\n",
    "# Convert to a list of letters list('abc') = ['a','b','c']\n",
    "letters = words.flatMap(lambda word: [pair[0]+pair[1] for pair in zip(list('_'+word),list(word+'_')) ] )\n",
    "counts = letters.map(lambda w: (w,1)).reduceByKey(lambda a,b: a+b)\n",
    "bigrams = counts.collect()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 59,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "e_ 172904\n",
      "_t 128820\n",
      "th 106824\n",
      "s_ 102536\n",
      "he 101810\n",
      "d_ 99436\n",
      "_a 97104\n",
      "t_ 92419\n",
      "_s 70618\n",
      "_h 67740\n",
      "in 66176\n",
      "an 64789\n",
      "n_ 63768\n",
      "er 63475\n",
      "_w 56231\n",
      "r_ 53641\n",
      "re 51571\n",
      "_i 49690\n",
      "_o 48646\n",
      "y_ 47589\n",
      "nd 47353\n",
      "o_ 43783\n",
      "on 40815\n",
      "ou 40242\n",
      "en 40018\n",
      "at 39559\n",
      "ha 39329\n",
      "_b 38703\n",
      "_m 36304\n",
      "_c 35617\n",
      "hi 35323\n",
      "to 34885\n",
      "ed 33906\n",
      "is 33101\n",
      "_f 32939\n",
      "ng 32927\n",
      "es 32406\n",
      "_d 31593\n",
      "it 31263\n",
      "or 31073\n",
      "ar 29993\n",
      "g_ 28676\n",
      "_p 28551\n",
      "_l 28257\n",
      "as 28217\n",
      "st 28095\n",
      "te 27913\n",
      "le 27804\n",
      "f_ 26620\n",
      "se 26332\n",
      "nt 25081\n",
      "a_ 24600\n",
      "me 24069\n",
      "h_ 23940\n",
      "l_ 23436\n",
      "_n 23129\n",
      "ve 22985\n",
      "de 22728\n",
      "of 22601\n",
      "ll 22433\n",
      "ro 22372\n",
      "_r 21788\n",
      "ea 21727\n",
      "ne 21470\n",
      "al 21141\n",
      "_e 20797\n",
      "ti 19930\n",
      "ho 19908\n",
      "no 19710\n",
      "co 19261\n",
      "ri 19213\n",
      "ce 18764\n",
      "m_ 18050\n",
      "be 17483\n",
      "om 16762\n",
      "el 16694\n",
      "wh 16330\n",
      "ut 16321\n",
      "ch 16294\n",
      "_g 16157\n",
      "us 15636\n",
      "ot 15593\n",
      "ur 15501\n",
      "ma 15336\n",
      "wi 15255\n",
      "wa 15228\n",
      "sh 15094\n",
      "ad 14907\n",
      "ow 14861\n",
      "lo 14690\n",
      "li 14339\n",
      "et 14115\n",
      "ra 13995\n",
      "si 13900\n",
      "ee 13712\n",
      "ss 13601\n",
      "ai 13570\n",
      "il 13492\n",
      "so 13230\n",
      "ta 13222\n",
      "ie 13145\n",
      "_y 13101\n",
      "un 13033\n",
      "i_ 13028\n",
      "la 12942\n",
      "im 12930\n",
      "pe 12391\n",
      "fo 12378\n",
      "ly 12137\n",
      "rs 12107\n",
      "io 11988\n",
      "nc 11793\n",
      "w_ 11660\n",
      "ic 11593\n",
      "oo 11470\n",
      "yo 11447\n",
      "we 11411\n",
      "ca 11378\n",
      "sa 11265\n",
      "id 11237\n",
      "di 11228\n",
      "mo 11008\n",
      "ac 10779\n",
      "ol 10724\n",
      "pr 10553\n",
      "tr 10312\n",
      "em 10273\n",
      "ke 10134\n",
      "u_ 10083\n",
      "ir 9817\n",
      "ns 9760\n",
      "ld 9748\n",
      "do 9523\n",
      "os 9350\n",
      "ge 9282\n",
      "ul 9189\n",
      "gh 9145\n",
      "ov 8958\n",
      "_u 8930\n",
      "k_ 8838\n",
      "rt 8561\n",
      "ni 8529\n",
      "ay 8467\n",
      "pa 7959\n",
      "_v 7951\n",
      "mi 7838\n",
      "av 7789\n",
      "na 7767\n",
      "po 7746\n",
      "ec 7694\n",
      "fr 7595\n",
      "bu 7533\n",
      "am 7432\n",
      "ig 7412\n",
      "ry 7177\n",
      "rd 7164\n",
      "fi 7113\n",
      "ev 7073\n",
      "wo 7060\n",
      "vi 6722\n",
      "fe 6566\n",
      "ht 6535\n",
      "pi 6432\n",
      "fa 6344\n",
      "rr 6329\n",
      "tt 6256\n",
      "ts 6236\n",
      "go 6227\n",
      "su 6061\n",
      "pl 6040\n",
      "bo 5927\n",
      "_k 5921\n",
      "ag 5852\n",
      "ct 5845\n",
      "tu 5807\n",
      "ap 5804\n",
      "ei 5786\n",
      "ia 5640\n",
      "bl 5628\n",
      "ba 5541\n",
      "qu 5410\n",
      "ey 5342\n",
      "p_ 5293\n",
      "if 5289\n",
      "ep 5281\n",
      "my 5273\n",
      "sp 5241\n",
      "od 5183\n",
      "mp 5121\n",
      "iv 5116\n",
      "ru 5010\n",
      "ab 4893\n",
      "ck 4831\n",
      "ki 4821\n",
      "ex 4740\n",
      "op 4711\n",
      "dr 4704\n",
      "ug 4661\n",
      "ga 4569\n",
      "ak 4568\n",
      "ok 4548\n",
      "gr 4534\n",
      "_j 4451\n",
      "da 4419\n",
      "ue 4408\n",
      "oi 4406\n",
      "ew 4392\n",
      "au 4333\n",
      "up 4295\n",
      "ef 4275\n",
      "ty 4261\n",
      "tl 4168\n",
      "ff 4152\n",
      "rm 4146\n",
      "lu 4130\n",
      "ci 4011\n",
      "ye 4004\n",
      "ds 3957\n",
      "rn 3741\n",
      "_q 3715\n",
      "by 3654\n",
      "gi 3639\n",
      "br 3626\n",
      "pp 3586\n",
      "cr 3541\n",
      "ui 3523\n",
      "sc 3402\n",
      "uc 3400\n",
      "cl 3346\n",
      "vo 3309\n",
      "va 3202\n",
      "nn 3198\n",
      "ny 3176\n",
      "wn 3172\n",
      "v_ 3056\n",
      "aw 3000\n",
      "kn 2998\n",
      "lf 2978\n",
      "eg 2764\n",
      "ft 2736\n",
      "mu 2731\n",
      "oc 2666\n",
      "fu 2637\n",
      "hr 2631\n",
      "ls 2618\n",
      "hu 2610\n",
      "ms 2514\n",
      "cu 2506\n",
      "nk 2493\n",
      "nl 2473\n",
      "sk 2469\n",
      "eo 2418\n",
      "lt 2408\n",
      "rc 2387\n",
      "gu 2374\n",
      "um 2363\n",
      "hy 2356\n",
      "ud 2344\n",
      "mm 2336\n",
      "pt 2301\n",
      "pu 2250\n",
      "af 2208\n",
      "sm 2140\n",
      "rg 2128\n",
      "du 2102\n",
      "fl 2038\n",
      "bi 2034\n",
      "gl 1998\n",
      "ik 1975\n",
      "rk 1949\n",
      "ys 1932\n",
      "eu 1926\n",
      "ua 1923\n",
      "sl 1882\n",
      "dy 1870\n",
      "ju 1869\n",
      "mb 1859\n",
      "oa 1839\n",
      "ps 1828\n",
      "tw 1815\n",
      "uk 1815\n",
      "ob 1804\n",
      "rl 1797\n",
      "je 1785\n",
      "ks 1727\n",
      "ib 1720\n",
      "ip 1676\n",
      "dd 1651\n",
      "xp 1610\n",
      "iu 1510\n",
      "dl 1510\n",
      "c_ 1490\n",
      "nu 1486\n",
      "gn 1484\n",
      "ya 1471\n",
      "gs 1439\n",
      "cc 1431\n",
      "sw 1410\n",
      "lk 1376\n",
      "nf 1360\n",
      "yi 1355\n",
      "ub 1351\n",
      "oy 1257\n",
      "ws 1223\n",
      "tc 1219\n",
      "rv 1203\n",
      "og 1103\n",
      "jo 1099\n",
      "ze 1080\n",
      "nv 1052\n",
      "xt 1027\n",
      "x_ 1022\n",
      "eh 994\n",
      "oe 983\n",
      "ae 922\n",
      "z_ 862\n",
      "yt 861\n",
      "ez 856\n",
      "wr 829\n",
      "iz 823\n",
      "ph 822\n",
      "rf 820\n",
      "kh 818\n",
      "ja 811\n",
      "rp 809\n",
      "xc 780\n",
      "ml 772\n",
      "dg 770\n",
      "gg 759\n",
      "bs 744\n",
      "xi 718\n",
      "à_ 714\n",
      "zo 701\n",
      "ku 695\n",
      "eb 691\n",
      "ka 679\n",
      "lm 657\n",
      "sn 655\n",
      "uz 652\n",
      "lv 636\n",
      "uf 632\n",
      "ko 632\n",
      "rb 628\n",
      "é_ 626\n",
      "oh 623\n",
      "sd 620\n",
      "xe 606\n",
      "lp 593\n",
      "sy 584\n",
      "_à 578\n",
      "lw 553\n",
      "dé 545\n",
      "_é 537\n",
      "vn 532\n",
      "lr 529\n",
      "lc 526\n",
      "ah 519\n",
      "sb 514\n",
      "kl 499\n",
      "az 497\n",
      "ém 481\n",
      "py 481\n",
      "bb 471\n",
      "cy 467\n",
      "dn 451\n",
      "ix 449\n",
      "hm 445\n",
      "xa 440\n",
      "ek 439\n",
      "_x 437\n",
      "wd 433\n",
      "b_ 423\n",
      "dm 419\n",
      "ii 418\n",
      "dv 406\n",
      "nq 405\n",
      "ux 397\n",
      "wl 397\n",
      "rw 393\n",
      "eq 390\n",
      "sf 377\n",
      "uo 368\n",
      "uv 362\n",
      "tm 361\n",
      "bt 359\n",
      "mn 350\n",
      "kp 317\n",
      "hs 315\n",
      "rh 314\n",
      "té 306\n",
      "nb 305\n",
      "sq 303\n",
      "zi 296\n",
      "cb 290\n",
      "ré 283\n",
      "dj 282\n",
      "tf 275\n",
      "ée 269\n",
      "tn 262\n",
      "vs 261\n",
      "yb 246\n",
      "ér 242\n",
      "_z 240\n",
      "gt 238\n",
      "êt 238\n",
      "vr 235\n",
      "ym 234\n",
      "oj 228\n",
      "ét 225\n",
      "yl 224\n",
      "tz 222\n",
      "hn 220\n",
      "yp 218\n",
      "vl 215\n",
      "bj 214\n",
      "aj 214\n",
      "nj 208\n",
      "ox 208\n",
      "nm 207\n",
      "ky 205\n",
      "sé 202\n",
      "nh 202\n",
      "yr 201\n",
      "vy 201\n",
      "gy 198\n",
      "hl 197\n",
      "fs 196\n",
      "mf 189\n",
      "j_ 189\n",
      "za 187\n",
      "èr 178\n",
      "ép 175\n",
      "yc 174\n",
      "mt 171\n",
      "xx 169\n",
      "oz 169\n",
      "né 168\n",
      "éc 161\n",
      "nw 158\n",
      "és 158\n",
      "zh 154\n",
      "np 152\n",
      "cq 146\n",
      "yw 145\n",
      "ln 144\n",
      "cs 136\n",
      "zu 132\n",
      "_ê 130\n",
      "én 130\n",
      "pc 127\n",
      "iq 125\n",
      "fy 125\n",
      "cd 124\n",
      "xv 121\n",
      "là 118\n",
      "nr 115\n",
      "lq 114\n",
      "ax 113\n",
      "vé 112\n",
      "mê 110\n",
      "yf 110\n",
      "lg 109\n",
      "kw 108\n",
      "êm 108\n",
      "lé 107\n",
      "df 106\n",
      "él 105\n",
      "pf 103\n",
      "ej 102\n",
      "zz 102\n",
      "lb 102\n",
      "gm 101\n",
      "kr 101\n",
      "dk 100\n",
      "wb 97\n",
      "ès 97\n",
      "hk 97\n",
      "hf 96\n",
      "mé 96\n",
      "pé 93\n",
      "nx 92\n",
      "gé 91\n",
      "yn 90\n",
      "aî 90\n",
      "mr 90\n",
      "rè 87\n",
      "dw 85\n",
      "ît 85\n",
      "xh 85\n",
      "iè 85\n",
      "uh 83\n",
      "rz 82\n",
      "tp 82\n",
      "sg 80\n",
      "sv 79\n",
      "vu 77\n",
      "hé 76\n",
      "bm 74\n",
      "hd 72\n",
      "hb 71\n",
      "hc 71\n",
      "wk 69\n",
      "ié 67\n",
      "gd 66\n",
      "où 65\n",
      "ù_ 65\n",
      "zm 63\n",
      "wf 63\n",
      "âm 62\n",
      "yu 61\n",
      "sr 61\n",
      "cé 61\n",
      "hh 59\n",
      "év 59\n",
      "mc 59\n",
      "rq 59\n",
      "zl 59\n",
      "dh 57\n",
      "_â 55\n",
      "dt 54\n",
      "ég 54\n",
      "dq 53\n",
      "bv 52\n",
      "ao 52\n",
      "oq 52\n",
      "éd 51\n",
      "uy 51\n",
      "wy 50\n",
      "pm 49\n",
      "ût 48\n",
      "aa 47\n",
      "hw 47\n",
      "rê 47\n",
      "uj 47\n",
      "kc 47\n",
      "ôt 45\n",
      "pè 45\n",
      "cè 43\n",
      "ué 42\n",
      "lh 41\n",
      "cx 40\n",
      "yk 40\n",
      "âc 39\n",
      "lè 38\n",
      "nz 37\n",
      "ço 37\n",
      "iw 36\n",
      "ww 35\n",
      "zy 35\n",
      "nê 34\n",
      "sû 34\n",
      "ûr 33\n",
      "pw 32\n",
      "èn 32\n",
      "dè 32\n",
      "kb 32\n",
      "km 32\n",
      "èl 31\n",
      "yd 30\n",
      "wu 30\n",
      "lx 30\n",
      "xy 29\n",
      "fw 29\n",
      "ji 28\n",
      "bd 28\n",
      "pk 27\n",
      "dc 27\n",
      "kg 27\n",
      "zd 27\n",
      "ât 27\n",
      "rj 27\n",
      "lz 27\n",
      "fb 27\n",
      "pç 26\n",
      "tô 25\n",
      "râ 25\n",
      "aq 25\n",
      "tb 25\n",
      "bh 25\n",
      "yh 24\n",
      "dp 23\n",
      "éa 23\n",
      "èv 23\n",
      "xl 22\n",
      "éf 22\n",
      "bé 21\n",
      "éj 21\n",
      "hè 21\n",
      "wt 20\n",
      "kf 20\n",
      "_ô 20\n",
      "bw 20\n",
      "pg 20\n",
      "xu 20\n",
      "mè 20\n",
      "oû 19\n",
      "tê 19\n",
      "wc 18\n",
      "db 18\n",
      "gw 18\n",
      "jà 18\n",
      "rô 18\n",
      "tè 17\n",
      "tv 17\n",
      "èm 17\n",
      "eû 17\n",
      "pê 17\n",
      "çu 17\n",
      "zn 17\n",
      "xo 17\n",
      "fâ 17\n",
      "ôl 16\n",
      "hâ 16\n",
      "îl 16\n",
      "eç 16\n",
      "_î 16\n",
      "ô_ 15\n",
      "fé 15\n",
      "pb 14\n",
      "èc 14\n",
      "èt 13\n",
      "tk 13\n",
      "yz 13\n",
      "êc 13\n",
      "bn 13\n",
      "yé 13\n",
      "gè 12\n",
      "tg 12\n",
      "mw 12\n",
      "kv 12\n",
      "în 12\n",
      "éb 11\n",
      "yg 10\n",
      "pn 10\n",
      "éq 10\n",
      "aï 10\n",
      "lâ 10\n",
      "cô 10\n",
      "vè 9\n",
      "éi 9\n",
      "èd 9\n",
      "td 9\n",
      "wm 9\n",
      "ih 9\n",
      "cv 9\n",
      "rç 9\n",
      "vk 8\n",
      "ët 8\n",
      "oë 8\n",
      "hp 7\n",
      "fû 7\n",
      "êl 7\n",
      "fk 7\n",
      "kt 7\n",
      "xq 7\n",
      "cw 7\n",
      "sj 6\n",
      "nç 6\n",
      "mk 6\n",
      "vê 6\n",
      "yv 6\n",
      "kd 6\n",
      "sè 6\n",
      "cz 5\n",
      "ïr 5\n",
      "zè 5\n",
      "ça 5\n",
      "âg 5\n",
      "fn 5\n",
      "zv 5\n",
      "sz 5\n",
      "jr 5\n",
      "vô 4\n",
      "ûm 4\n",
      "èg 4\n",
      "zk 4\n",
      "hv 4\n",
      "ên 4\n",
      "lû 4\n",
      "ij 4\n",
      "fî 4\n",
      "ân 3\n",
      "ïs 3\n",
      "cm 3\n",
      "zs 3\n",
      "pâ 3\n",
      "ûl 3\n",
      "fh 3\n",
      "uï 3\n",
      "gf 3\n",
      "bc 3\n",
      "tâ 3\n",
      "mv 3\n",
      "pû 3\n",
      "rû 3\n",
      "ï_ 3\n",
      "dû 3\n",
      "uê 3\n",
      "pd 3\n",
      "èb 3\n",
      "uw 3\n",
      "éu 3\n",
      "vg 3\n",
      "âl 3\n",
      "êv 3\n",
      "uq 3\n",
      "éé 2\n",
      "vv 2\n",
      "tx 2\n",
      "îc 2\n",
      "dî 2\n",
      "aç 2\n",
      "ôn 2\n",
      "gz 2\n",
      "û_ 2\n",
      "gê 2\n",
      "_è 2\n",
      "oï 2\n",
      "îm 2\n",
      "bâ 2\n",
      "wp 2\n",
      "oî 2\n",
      "gâ 2\n",
      "éo 2\n",
      "ïe 2\n",
      "xé 2\n",
      "uu 2\n",
      "fm 2\n",
      "hê 2\n",
      "dz 2\n",
      "gb 2\n",
      "èq 2\n",
      "vt 2\n",
      "bî 2\n",
      "dâ 2\n",
      "oé 2\n",
      "ûc 1\n",
      "ûn 1\n",
      "cn 1\n",
      "xm 1\n",
      "vw 1\n",
      "bf 1\n",
      "ïl 1\n",
      "q_ 1\n",
      "éh 1\n",
      "câ 1\n",
      "mô 1\n",
      "mg 1\n",
      "iy 1\n",
      "pô 1\n",
      "fp 1\n",
      "fè 1\n",
      "xs 1\n",
      "nû 1\n",
      "rî 1\n",
      "hq 1\n",
      "bû 1\n",
      "mâ 1\n",
      "kk 1\n",
      "fê 1\n",
      "cê 1\n",
      "cg 1\n",
      "iâ 1\n",
      "mq 1\n",
      "âb 1\n",
      "nâ 1\n",
      "wg 1\n",
      "ïv 1\n",
      "éâ 1\n",
      "zb 1\n",
      "ôd 1\n",
      "oê 1\n",
      "hg 1\n",
      "zw 1\n"
     ]
    }
   ],
   "source": [
    "for r,c in sorted(bigrams,key=lambda x:x[1],reverse=True):\n",
    "    print r,c\n",
    "    \n",
    "    \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 129,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "16694"
      ]
     },
     "execution_count": 129,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bigrams[0][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAqgAAAKoCAYAAAC7uA1cAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAAPYQAAD2EBqD+naQAAIABJREFUeJzs3XmYXHWd7/HPN70knaTTCQkQlhAMEGTR0GmWixAJy0UZ\nYJSQrVBhHBnHmRG4yQ1hGcBBGBCIZC5473gVZQaQivCIjvg8OgxD4II6snQAkZ0AsptA0umsvX3v\nH6dau5ssVd3nV+dXqffrefJIn5zz6a+VqupPn6WOubsAAACAWAzLegAAAACgLwoqAAAAokJBBQAA\nQFQoqAAAAIgKBRUAAABRoaACAAAgKhRUAAAARIWCCgAAgKhQUAEAABAVCmrkzGy5md2Y9RxpqaT/\nP2b2HTN738y6zezjWc+zPeV4XEN/DzO71czuSTGv4h8TAKhWtVkPAMTIzD4t6WxJx0l6VdLqbCeK\nwhmSOgPmny/JAuYDKJKZLZe0wt0XZj0LqhMFFdi6/SW94+6/yXqQWLj72sD57SHzgRiYWZ27h/xF\nD9gpcIg/JWb2KTN72MzWmNlqM7vXzKakFF9rZjeb2VozW2VmX08pV5ZYbGYvmdlmM3vNzC5JKXuk\nmd1mZu1m9paZpfabeGHuS8xspZltNLMVZnZmStm3SrpJ0j5m1mNmK9PILWSPNrMfmNl6M3vDzM5L\n8TDxMDO7rnBawjtm9rUUMv+o0g7xbyX/1MJrKBfqewxW4bG9ycyWmtkHZvaumX2p8Br6vpmtK7xG\nP53C9/lfIZ4nZlZf+P/wnpltKrwfHp5S9vLCe2Dq74Nbe59K87neZ/alZrZK0i/SyC1kzzazpwvv\ngavN7D4za0gh91YlR48uKLwHdpvZPinkvmpm5w9YtsLMrkgh+6/M7K2tLP83M7tlCLmnFn6uW+Hr\naYXH5Jo+69xiZrcNMn9C4XV4cZ9lnzCzLWZ2/GDnLuR8ofC8qBuw/Cdm9q9DzJ7c57nR0+fPA0PJ\n7YuCmp5Rkr4pabqkEyR1S/pxStl/oeTQ6hFKDoMuNLMvpZT9DUmLJV0p6SBJ8yS9m1L2EkkzJJ0u\n6WRJM5U8Pmm4VNLnJX1Z0sGSlkq63cxmpJB9vqQrJL0paXclj3talko6WtJpkj6l5DFpTin7HEnr\nJR2p5N/0CjM7MaXsimZmZ0n6gaScu+eznmcbzpa0Ssnz7SZJ35Z0t6RfKnmO3CfpNjMbkcL3CfE8\nuUHJaSBfUDLvy5L+3czGppAtJXOHeB/c2vtUWq/JXmdL2iLpE5K+kkagmU2UdKekWyR9VEmhvEfp\nnCZzgaRfS/qukvfAPSS9kUJuSHdL2qVvqTOzcUreZ+8YQu7DkkbrT8+J45S8Tmf2WeeTkpYPJtzd\nV0v6S0lXmtl0Mxst6TZJN7n7oDL7uFtJz/vz3gVmtqukP5P0vSFm/17SRCXPjYlKHp/3JT00xNw/\ncXf+BPgjaYKkHkkHDzFnuaRnBiy7duCyQWaPlrRJ0hcD/P8fJWmzpFl9lo2TtEHSjUPMrlfyA/ao\nAcu/K+mOlOa/QNLKlB+T0Up+SJ3RZ9mYwv+XoT4myyU9NGDZbyRdk+L8y4c65w7yb5V0T9rzSvpb\nSR9IOjbAzKk8JgP//ZT8UGmX9C99lu1eeE85MrbniaSRhef2vD7LapX8kvc/U3p8Un8fDPk+NWD2\nxwM895qV7AiZlHZ2n7lTfb0rOZ///AHLVki6IqX8H0v6bp+vvyzpjRRyH5e0sPDf90i6uPCzc6Sk\nvQqvy/2G+D1ulvS8kjL9pKS6lB6T/y3pZ32+XijppZT/XYdL+i9JP0kzlz2oKTGz/c3sTjN7xcza\nlLwQXdKQD4so+Yfv69eSDug95DAEBykpe6ntku9jP0l1kh7tXeDuayS9kEL2/kreGP6jcFiu3cza\nley52S+F/FCmKPmh/VjvAndfp3QeE0l6esDX70jaLaXsSjVHSUn97+7+SNbD7MAf//3cvUfJ3ojf\n9ln2XuE/h/pvGuJ5sp+S5/avehe4e5eS1/9BQ8zuFeJ9MOT7VF9PpJwnSU9J+k9Jz5jZXWZ2bop7\nqyvVDySd2eeQ9lmSlqWQ+5D+tMd0hpKS+pykY5XsPX3L3V8Z4ve4UMlraLakszy985S/K+lkM9uj\n8PU5SnYGpOlWJb/sfS7NUApqen6m5Dfvc5UcOjtSyaGW+iyH2oFNWQ8wSKML//tnkqb1+XOwkhd3\ntRr4hubiNd6q5HBcWqfEhLS1f7+t/ZAa6r8pz5Py25B2oLv3uPvJkj4t6XeSzpP0vJlNTvt7pahH\nHz4FoW5rKw7SvUqey6ea2d5KyuRQDu/3elDSsWY2TVKHu7+opLQer+SQfxqHtfeXtKeS+T+SQp4k\nyd2fVPJL6dlmNl3Jz8khnX/al5ldJum/Szrd3VN9nvOmlAIz20XSVElXu/tyd39B0vgUv8VRA74+\nWskueh9i7ktKDm+FOE/xFUld6jN74XygqSlkP6vkcOJkd1854M+HTpKPyEolj8kfz2k1syal85hg\n615R8kPkM2Z2c9bD7MReUVJ8j+ldYGa1Sp7rv0vpe4R4Hwz5PlUW7v5rd79SySH/TiXnAaehQ1JN\nSlm9Vik5Z1GSZGZjlG4Z26Jk7+bnJeUkPe/uT6UQ/bCS07EW6E9l9EEle1WPK/z3oBX2+N6uZG/v\n5ZK+Z2YThpI5wC2Svlj4c39aPyctuTD5Mklz3P21NDL74mOm0rFGyeG4L5vZu5ImKzk/aqgFstc+\nZrZE0ncktUj6qpIXypC4+xYzu07S9WbWqeRijF0lHeLu3x9i9gYz+56kG8zsAyVvTFcrOWdqqHOv\nLzweS82sRtIjkpqU/HBsc/fbh/o9QijM/a+SlpjZGiWPyT8oeUzSeq5gAHd/uXDhxHIz63L3Ib92\n0J+7bzSzf1byel+j5IKaxZIaJA3pvaSP1N8HQ75PhWZmRyrZuXCfpD9I+m9Krn14NqVv8Zqkowp7\nZNdL+iCFnSIPSDrHzH4mqU3JxbldQ8wc6AdKjmgeoqT0DZm7rzWzp5Ucwv67wuL/J+kuJT1qqHtQ\nr1FSgM+TtFHJ0cFblVy4l4Y7lVwMeK6SU+GGzMwOUbIn9jpJz5nZ7oW/6iicJjNkFNQUuLub2Twl\nV97+Vsn5S+driL9V9cYruaKvQcl5Ul2Slrr7oD82o1+4+9cL5fRKJYcX3lFy9XAaLlRyXspPlVzw\n8U0lL8Ihc/fLzewPSk5WnyJprZLDuddsd8PsLVDy+N4raZ2k6yVNUrIneyjKUXArrUT/cV53f7Fw\npXpvSb0w7e8RIKfYZUP9Pmm5WMnh29skNSq5sORkd29LKT/U++DW3qeaUsjtFeoxX6fk/McLlLyv\nvq7kQp77UspfIulflBTeEUr2dP5+iJnXStpXyftfm5K9hfsOMXOgB5RcFHmAkmKWloeUnEr2oJSc\nq2xmz0ra1d1fGmyomR2npC/M7D1EbmZnS3rSzP7a3f/vUAd393Vm9iMlxfffhppXcLiS1+NlhT+9\nHlLySUZDZkP/hQjAYJnZSElvKfnBkvaJ6xXFzO6U1OXuZ2c9C+JiZb6rUbm/HxCamd0v6beVdASp\navagmtkkJb8Juj58krYr+TioN8s+GKqKmR2m5DMLH5U0VsnnrbrS+6224hRO0zhQyTmFae29B4Cq\nV/hkh96Luf4m43FKUjUFVdLbSnbPb+/vgXJYpOQijA4lHz9zrLt/kO1ImTpUyccT/acoqNi6ch/q\n49AidhYrlOwMWTyUUxGywCF+AAAARIWPmQIAAEBUKKgAAACICgUVAAAAUdkpLpIys/GSPqXkg4WH\n+nmSAAAASN8IJZ99++/u/v72VtwpCqqScvqDrIcAAADADn1OO7iRws5SUF+TpDvuuEMHHXTQDlde\nsGCBli5dWnT4mjXF37Xr8ssv11VXXVX0+k1Nxd+wZOHChbrxxhuLXr+np6fodRctWqQlS5YUvf51\n111X9Lq/+tWv9IlPfKLo9Zubm4te94477tDnP//5otevrS3+KX/bbbfp7LOL/8z4TZs2Fb3usmXL\nNH/+/KLXL+Xx+9rXvqYrr7yy6PVHjRpV9LoXX3yxvvGNbxS9fk1N8bfyXrx4sa6//vqi11+xYkXR\n637729/WV77ylaLX32OPPXa8UsE111yjSy+9tOj1t2zZUvS6N9xwgy68sPibXq1fv77odb/1rW/p\nq1/9atHrT5hQ/K3BS31Mxowp/gZzl112ma6++uqi1x87dmzR65b6s6GU5+DNN9+s8847r+j1X3nl\nlaLXvfvuuzVnzpyi199rr72KXrfU187UqVOLXveKK67Q17/+9aLXL+X9pNTnSXt7e9Hr/uM//qP+\n/u//vuj1u7uLv3Putddeq0suuaTo9b/1rW8Vve7TTz+tj3/840Wv/7nPfa7odYt9fr/++uu9Hem1\nHa27sxTUzZJ00EEHafr06Ttcuampqaj1eq1atarodceMGVPSE2D8+PFFr1vq3KUU1FKzd91116LX\nHT58eEnr77vvvkWvO3LkyJLWr6+vLyl7ypQpRa9fSjloaGjQ5MmTi16/lOdUqc/BxsbGkrIPO+yw\notcv5ReCpqamkn45KeXxHjVqlA444ICi1y/l36axsVGHHHJI0euX8otMY2NjUb9091q3bl3R644a\nNUoHHnhg0etPnDix6HVLfUx22WWXotcdM2aMpk3b3kda9xfyPXbjxo1Frzt69OiSHu/Ozs6i121o\naNA+++xT9PqlvK+NHj26pNdOyPeqUt5PSn2erF27tuh1Gxsbdeihhxa9fldXV0nZpbx2xo0bV/S6\ndXV1Ja1fyvO11Oe3ijgdk4ukAAAAEBUKKgAAAKJCQQUAAEBUqrKg5nK5YNlnnHFGsOxSLqop1bx5\n84Jl77///sGyjz766GDZxxxzTLDso446Klj2Zz/72WDZpVyIEVP28ccfHyz7tNNOC5b96U9/Olj2\nSSedFCw75GMya9asYNkhfzaEfLyPOOKIYNkzZ84Mlh3y52XI58npp58eLPvUU08Nlj1p0qRg2SGe\n3+buqYeWm5lNl/TEE088UdIJ7sUq5SKpUpVyAn+pSrlIqlSlXP1bqmOPPTZYdikXSZWqlIt2SnXK\nKacEyy7lIqlSlXJRQ6kefvjhYNmlXCRVqlIukipVKRdJlaqUi6RKVcpFUqUK+R77yCOPBMt+5pln\ngmWXcpFUqUq5MKlUId9PSrlIqlSlXCRVqlI+LaNUCxYsSD3zhRde0LnnnitJLe7eur11q3IPKgAA\nAOJFQQUAAEBUKKgAAACICgUVAAAAUdlZ7iQlKTmhd+CtQ3O5XNArMwEAANDf/fffr/vvv7/fslIu\nJt6pCurSpUuDXMUPAACA4p100kkf+vipPlfx7xCH+AEAABAVCioAAACiQkEFAABAVKI8B9XMJkl6\nVpJLsgF/7ZIOdvc3yz4YAAAAgouyoEp6W9L27pf2drkGAQAAQHlFWVDdvVvSyqznAAAAQPlxDioA\nAACiQkEFAABAVKI8xD9Y7733nt58M/1rp/bee+/UM3u9/Xa402m7urqCZd90003Bsjs7O4Nlb9q0\nKVh2XV1dsOy33norWPaECROCZb/++uvBsmfOnBkse+3atcGye3p6gmUfeeSRwbLr6+uDZYd43+41\nfvz4YNkf+9jHgmU3NzcHyzYbeO1xekI+T0K+Ljds2BAse5999gmWvWzZsmDZW7ZsCZZdDPagAgAA\nICoUVAAAAESFggoAAICoUFABAAAQFQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRibagmtmnzOxh\nM1tjZqvN7F4zm5L1XAAAAAgr2oIqaZSkb0qaLukESd2SfpzpRAAAAAgu2luduvs9fb82s3Ml/cHM\nDnb3ZzMaCwAAAIFFuwfVzPY3szvN7BUza5P0qiSXFO6mtgAAAMhctHtQJf1MSSk9V9LbSsr07yTV\nb2uDK6+8Uo2Njf2WfeYzn9FnP/vZgGMCAACgr7vvvlt33313v2Xr1q0revsoC6qZ7SJpqqQvufsv\nC8uO3dF2X/va1/Sxj30s9HgAAADYjjlz5mjOnDn9lj355JOaMWNGUdtHWVAlrZH0vqQvm9m7kiZL\nulbJIX4AAADsxKI8B9XdXdI8SS2Sfqvkav5FmQ4FAACAsoh1D6rc/QFJhw5YXJPFLAAAACifKPeg\nAgAAoHpRUAEAABAVCioAAACiQkEFAABAVCioAAAAiEq0V/EPxpgxYzRu3LjUc7u6ulLP7DVx4sRg\n2cOGVebvH/X127xZ2JCNGjUqWHZITU1NwbKTT3ULY//99w+WHdKuu+4aLDvEe1Sv2tpwb+khnyeT\nJk0Klh1SyNdlperu7g6WPWHChIrMrlR1dXWpZ44cObLodSuzwQAAAGCnRUEFAABAVCioAAAAiAoF\nFQAAAFGhoAIAACAqURZUM1tuZjdmPQcAAADKL8qCCgAAgOpFQQUAAEBUYi6otWZ2s5mtNbNVZvb1\nrAcCAABAeDEX1L+Q1CnpCEnnS1poZl/KdCIAAAAEF/OtTn/v7gsL//2SmX1c0gJJ38twJgAAAAQW\nc0H9rwFf/1rJXlTzbdwU+qKLLvrQvZFnz56tuXPnBhoRAAAAA+XzeeXz+X7L2trait7ettH1MmVm\nyyW94u7n9ln255LuljRiYEE1s+mSnnjkkUd02GGHpT7P8OHDU8/sNWxYuLMsQmZj5xHyPcDMgmVX\nqq6urmDZtbXh9jnwPEExuru7g2XX1NQEy0Z5tLa2qqWlRZJa3L11e+vG3GCOGvD10ZJe2tbeUwAA\nAOwcYi6o+5jZEjObamY5SV+V9E9ZDwUAAICwYj0H1SXdJqlB0qOSuiQtdfdbMp0KAAAAwUVZUN39\nhD5f/l1mgwAAAKDsYj7EDwAAgCpEQQUAAEBUKKgAAACICgUVAAAAUaGgAgAAICpRXsU/FCHuSNLZ\n2Zl6Zq+GhoZg2ZV6R49KvdNOpdq4cWOw7FGjRgXLrlQhn4MbNmwIlh3yvapS7yTV09MTLDvkPWkq\n9f27Uu8k9cYbbwTLnjRpUrDsrLEHFQAAAFGhoAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUKKgAA\nAKJCQQUAAEBUKKgAAACIStQF1RKLzewlM9tsZq+Z2SVZzwUAAIBwYr+tzjckfUnS/5D0S0m7STo4\n04kAAAAQVLQF1cxGSzpf0t+6+x2Fxa9K+k12UwEAACC0aAuqpIMk1Ut6oNgNLrroIjU1NfVbNmfO\nHM2dOzfl0QAAALAt+Xxe+Xy+37K2trait4+5oG4qdYPrrrtOzc3NIWYBAABAkXK5nHK5XL9lra2t\namlpKWr7mC+SeknSZkknZj0IAAAAyifaPajuvsXMrpN0vZl1KrlIaldJh7j797OdDgAAAKFEW1Al\nyd2/XiinV0raU9I7kr6d7VQAAAAIKeqCKknufq2ka7OeAwAAAOUR8zmoAAAAqEIUVAAAAESFggoA\nAICoUFABAAAQFQoqAAAAohL9VfylMDOZWeq53d3dqWeWQ01NTbDsnp6eYNnr1q0Llr3LLrsEy8bO\nY+3atcGyx44dGyx75cqVwbIPOuigYNnDhlXmvpKOjo5g2SNGjAiWHVKl/lu6e7DsSZMmBcsOKcRj\nUkpmZT6TAAAAsNOioAIAACAqFFQAAABEhYIKAACAqFBQAQAAEJWKKahmttzMbsx6DgAAAIRVMQUV\nAAAA1YGCCgAAgKhEWVDNbKSZ3WZm7Wb2lpktzHomAAAAlEeUBVXSEkkzJJ0u6WRJMyVNz3IgAAAA\nlEd0tzo1s1GS/lLSWe7+YGHZOZLezHIuAAAAlEd0BVXSfpLqJD3au8Dd15jZCzvacPHixWpqauq3\nbO7cuZo7d27qQwIAAGDr8vm8li1b1m/Z2rVri94+xoI6aNdff72am5uzHgMAAKCq5XI55XK5fsta\nW1t1+OGHF7V9jOegviKpS9JRvQvMbJykqZlNBAAAgLKJbg+qu28ws+9JusHMPpC0StLVkrqznQwA\nAADlEF1BLbhQ0ihJP5XULumbksZkOhEAAADKIsqC6u4bJJ1T+NPrmxmNAwAAgDKK8RxUAAAAVDEK\nKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUYnyKv7BqqmpUU1NTeq5DQ0NqWf2+tGPfhQse9999w2W\nPXbs2GDZe++9d7DsVatWBcvesGFDsOyf/vSnwbK/+MUvBssOObe7B8v+yEc+Eiz7qaeeCpZ9yCGH\nBMuurQ334yLkYzJt2rRg2a2trcGyJ0+eHCy7vr4+WPaoUaOCZV944YXBss8777xg2aNHjw6WHbKf\nPPLII6lnvvzyy0Wvyx5UAAAARIWCCgAAgKhQUAEAABAVCioAAACiQkEFAABAVCioAAAAiAoFFQAA\nAFGhoAIAACAq0RZUS1xiZivNbKOZrTCzM7OeCwAAAGHFfCepSyWdJenLkl6W9ElJt5vZH9z94Uwn\nAwAAQDBRFlQzq5d0iaQT3f03hcWvmdkMSX8tiYIKAACwk4qyoEraX9JISf9hZtZneZ2kFdvaaNGi\nRWpqauq3bN68eZo3b16QIQEAAPBhDz74oB566KF+yzZu3Fj09rEW1NGF//0zSW8P+Lst29poyZIl\nam5uDjYUAAAAdmzmzJmaOXNmv2Uvv/yyLrjggqK2j7WgPqukiE5290eyHgYAAADlE2VBdff1ZrZE\n0lIzq5H0iKQmScdIanP32zMdEAAAAMFEWVAlyd0vN7M/SLpY0hRJayW1Srom08EAAAAQVLQFVZLc\n/WZJN2c9BwAAAMon2g/qBwAAQHWioAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBVz96xnGDIzmy7p\niV/+8pdB7iQ1YsSI1DN79b+TKySpp6cnWPaGDRuCZTc2NgbLDqmzszNYdl1dXbDskLq6uoJl19aG\n+/CUkO/nH3zwQbDs8ePHB8tGeYV8DvLzsvK1traqpaVFklrcvXV767IHFQAAAFGhoAIAACAqFFQA\nAABEhYIKAACAqFBQAQAAEJWKKahm9h0ze9/Mus3s41nPAwAAgDDCfd5Jiszs05LOlnScpFclrc52\nIgAAAIRSEQVV0v6S3nH332Q9CAAAAMKKvqCa2a2SzpHkZtYj6TV3n5LxWAAAAAgk+oIq6XxJr0j6\nK0mHSwp3myEAAABkLvqC6u7tZtYuqdvdV2U9DwAAAMKKvqCWYvHixWpqauq3bO7cuZo7d25GEwEA\nAFSffD6vfD7fb1lbW1vR2+9UBfX6669Xc3Nz1mMAAABUtVwup1wu129Za2urWlpaitq+Yj4HFQAA\nANWBggoAAICoUFABAAAQlYooqO7+v/jsUwAAgOpQEQUVAAAA1YOCCgAAgKhQUAEAABAVCioAAACi\nQkEFAABAVHaqO0nV1taqrq4u9dzu7u7UM3vV1lbmP4G7B8s2s2DZjY2NwbJ7enqCZQ8bFu53yZBz\nV6pKfV3+/ve/D5Y9efLkYNmVqlJf8yFt2rQpWPbIkSODZSM+lfkKAAAAwE6LggoAAICoUFABAAAQ\nFQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRoaACAAAgKlEWVDMbbWY/MLP1ZvaGmZ1nZsvN7Mas\nZwMAAEBYURZUSUslHS3pNEmfkjRTUnOWAwEAAKA8orufn5mNlnS2pPnu/mBh2RclvZ3lXAAAACiP\n6AqqpClK5nqsd4G7rzOzF3a04aJFi9TU1NRv2bx58zR//vzUhwQAAMDW5fN55fP5fsva2tqK3j7G\ngjpoS5Ys0fTp07MeAwAAoKrlcjnlcrl+y1pbW9XS0lLU9jGeg7pSUpekI3oXmFmTpKmZTQQAAICy\niW4PqruvN7N/lbTEzNZIWiXpHyR1S/IsZwMAAEB4Me5BlaQFkn4l6V5J90l6RNLzkjZnORQAAADC\ni7KguvsGd/+Cuze6+16SvivpQEkvZzwaAAAAAovuEL8kmdlhkj4q6VFJYyVdoeTw/r9lORcAAADC\ni7KgFixScmFUh6QnJB3r7h9kOxIAAABCi7KguvuTkg7Peg4AAACUX5TnoAIAAKB6UVABAAAQlSgP\n8Q/W+vXrS7qNVrHGjx+femY5PPPMM8GyDz300GDZleqNN94Ilj158uRg2cOHDw+WHVJHR0ew7Pr6\n+mDZIYV8noTU3d0dLLumpiZY9rBh4fbxhPhZ1uuBBx4Iln3GGWcEyw6ptbU1WPa0adOCZYd8fm/c\nuDH1zE2bNhW9LntQAQAAEBUKKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUYmyoJrZcjO7Mes5AAAA\nUH5RFlQAAABULwoqAAAAohJzQR1mZteZ2ftm9o6ZfS3rgQAAABBezAX1HEnrJR0pabGkK8zsxGxH\nAgAAQGgxF9Sn3f0qd3/F3W+X9LgkCioAAMBOrjbrAbbj6QFfvyNpt+1tcNlll2nMmDH9ls2aNUtn\nnnlmyqMBAABgW+666y7dfffd/Za1tbUVvX3MBbVzwNeuHezxvfrqqzVt2rRwEwEAAGCH5s6dq7lz\n5/ZbtmLFCh177LFFbR/zIX4AAABUIQoqAAAAohJrQfWsBwAAAEA2ojwH1d1P2MqyM7KYBQAAAOUV\n6x5UAAAAVCkKKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUYnyKv7Bqq2tVV1dXeq5GzZsSD2z17Bh\n4X5HmDJlSrDsNWvWBMseeLvaNK1evTpY9vjx44Nlv/jii8Gyp06dGizbPdwnxr3++uvBskM+B2tq\naoJljx49Olh2iPfWXqtWrQqWPXHixGDZIecePnx4sOwTTvjQB+WkZvPmzcGyV65cGSy7tjZcHXr3\n3XeDZe++++7Bsjdu3Jh6ZinPD/agAgAAICoUVAAAAESFggoAAICoUFABAAAQFQoqAAAAolIxBdXM\nlpvZjVnPAQAAgLAq6WOmzpDUmfUQAAAACKtiCqq7r816BgAAAITHIX4AAABEpWIKKgAAAKoDBRUA\nAABRqZhzUItxySWXqKmpqd+y2bNna/bs2RlNBAAAUH3uuece3XPPPf2WrVu3rujtd6qCeu211+qw\nww7LegyLztFgAAAgAElEQVQAAICqNmvWLM2aNavfsqeeekonnXRSUdtziB8AAABRoaACAAAgKpVU\nUD3rAQAAABBexZyD6u4nZD0DAAAAwqukPagAAACoAhRUAAAARIWCCgAAgKhQUAEAABAVCioAAACi\nUjFX8Rdj1KhRamxsTD3XzFLP7OUe7tOzQs49cuTIYNkh7b777lmPMChTp07NeoRBCfkcPOCAA4Jl\no7wmTpyY9QiDsuuuu2Y9QlU5+OCDsx6hqkyYMCH1zHHjxhW9LntQAQAAEBUKKgAAAKJCQQUAAEBU\nKKgAAACICgUVAAAAUamYgmpmt5rZPVnPAQAAgLAq6WOmzpcU7jNrAAAAEIWKKaju3p71DAAAAAiP\nQ/wAAACISsUUVAAAAFQHCioAAACiUjHnoBZjwYIFGjt2bL9l8+fPVy6Xy2giAACA6pPP55XP5/st\na2trK3p7c/e0ZwrCzG6V1OTus7byd9MlPfH4449r+vTpIb536pm9Qj7+IecGAAAoRWtrq1paWiSp\nxd1bt7cuh/gBAAAQFQoqAAAAokJBBQAAQFQqqaAOl7Q+6yEAAAAQVvQF1cxqzOxgSUdL+l3W8wAA\nACCs6AuqpEMlPSbpt5K+nfEsAAAACCz6z0F196ckjcp6DgAAAJRHJexBBQAAQBWhoAIAACAq0R/i\nj0F3d3ew7JqammDZ7e3twbIbGxuDZaO8uJvZh/X09ATLHjYs3H6BkHOHfJ6EfB8MKeTjHVLI52BX\nV1ew7NpaKks1YQ8qAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRoaACAAAgKtEWVDNbbmY3Zj0HAAAA\nyivaggoAAIDqREEFAABAVCqmoJrZqWa21sxyWc8CAACAcCritgxmdpak/yMp5+4/z3oeAAAAhBP9\nHlQz+1tJ35J0GuUUAABg5xf7HtQ5knaVdIy7P7GjlRcsWKCxY8f2WzZ//nzlcpwVAAAAUC75fF75\nfL7fsra2tqK3N3dPe6ZUmNlySeskTZd0r7v/7XbWnS7piccff1zTp09PfZaenp7UM3vV1NQEy25v\nbw+W3djYGCwb5RXyPcDMgmWHFPI1P2xYuANXIecO+TwJ+T4YUsjHO6SQz8Gurq5g2bW1se9Tw460\ntraqpaVFklrcvXV768Z+iP8VScdL+oyZ3Zz1MAAAAAgv+l9H3P1lMzte0nIz63L3BVnPBAAAgHBi\nLqh/PJ7k7i+a2Yn6U0m9MMO5AAAAEFC0BdXdTxjw9fOS9shoHAAAAJRJ7OegAgAAoMpQUAEAABAV\nCioAAACiQkEFAABAVCioAAAAiEq0V/EPRkdHh7Zs2ZJ67qZNm1LPLEf2uHHjgmWHvINKiH/DXpV6\nx5r6+vpg2evXrw+WHfKOY93d3cGyX3311WDZU6ZMCZYdUkdHR7DskHe922233YJlhxTyLmwhXztr\n164Nlj1+/Phg2SFV6h31QtwVrJRM9qACAAAgKhRUAAAARIWCCgAAgKhQUAEAABAVCioAAACiUjEF\n1cyWm9mNWc8BAACAsCqmoAIAAKA6UFABAAAQFQoqAAAAokJBBQAAQFQoqAAAAIhKbdYDpGnx4sVq\namrqt2zOnDmaN29eRhMBAABUn2XLlumHP/xhv2VtbW1Fb79TFdTrr79ezc3NWY8BAABQ1ebPn6/5\n8+f3W9ba2qqjjjqqqO05xA8AAICoUFABAAAQlUoqqJ71AAAAAAivYs5BdfcTsp4BAAAA4VXSHlQA\nAABUgUHvQTWzOkkTJY2UtMrdP0htKgAAAFStkvagmlmjmf2NmT0kaZ2k1yQ9J2mVmb1uZt81syMC\nzAkAAIAqUXRBNbOFSgrpFyXdL+mzkg6TNFXS0ZKuVLJH9j4z+4WZHZD6tAAAANjplXKI/whJn3T3\n323j7x+V9H0z+4qSEjtD0ktDnA8AAABVpuiC6u65ItfbIunbg55oCHp6etTT05N67qhRo1LP7DVu\n3Lhg2ZWqtjbch0vU1dUFy65UZpb1CINSU1MTLHu//fYLlt3Z2Rksu76+Plh2yLl32223YNkhhfh5\n02vYsHDXMId87Qy83XiaKvW9CoPDVfwAAACISqoF1cz2M7MH0swEAABAdUl7D+poScelnAkAAIAq\nUtLJfmZ2/g5W2WsIswAAAAAlf1D/P0l6R1LHNv4+3Bn6AAAAqAqlFtTXJV3k7ndt7S/N7DBJTwx5\nKgAAAFStUs9BfUJSy3b+3iXxORAAAAAYtFL3oF4haeR2/v5ZSR8Z/DgAAACodiXtQXX3Z9398e38\nfae7v15KppktN7ObzGypmX1gZu+a2ZfMbKSZfd/M1pnZS2b26VJyAQAAUJmG/DFTZnaxmY0dYszZ\nklYpuZ3qTUruRHW3pF9KapZ0n6TbzGzEEL8PAAAAIpfG56BeKmmXIWY85e7XuPsrkr4habOkVe7+\nvcKyr0uaIOnjQ/w+AAAAiFwaNz1P46Kop3v/w917zOx9Sb/ts+y9wj14t3vD5osuuuhD9wGeM2eO\n5s6dm8KIAAAAKMayZcv0wx/+sN+ytra2ordPo6CmoXPA176VZdIO9vhed911am5uTm0oAAAAlG7+\n/PmaP39+v2Wtra066qijito+jYJ6sKS3U8gBAAAABncOqpmtNLPxkuTub7h7d2H5WDNbmeaAAAAA\nqC6DvUhqX0k1W1k+XNJeJWb5EJYBAABgJ1PSIX4z+/M+X37KzPqe7Voj6URJr5WS6e4nbGXZlK0s\n21ohBgAAwE6m1HNQf1L4X5f0rwP+rlNJOf2fQ5wJAAAAVaykguruwyTJzF6VdIS7rw4yFQAAAKrW\noK7id/ePpD0IAAAAIJVwkZSZzd/xWn9cd5KZHTO4kQAAAFDNSrmK/2/M7DkzW2xmBw38SzNrMrM/\nM7M7JbVKGp/alAAAAKgaRR/id/fjClfxnyfpWjPbIOk9SZsljZM0UdJqSf8i6VB3fy/9cbevtrZW\ntbXp3xzrySefTD2z15FHHhksu1KtXbs2WPauu+4aLLuzc2s3P0tHXV1dsOzHHnssWPbxxx8fLPu9\n98K9xey+++7Bsuvr64Nld3d3B8tetWpVsOwxY8YEy65Uw4YN9lMgs/Xuu+8Gy95zzz2DZXd0dATL\nbmhoCJYdUk1N+h+eVEpmqRdJ/VTST81sgqQZkvaR1KCkmK6QtMLde0rJBAAAAPoa7O7Gj7n7j7f2\nF2b21+7+f4cwEwAAAKrYYI8h/MLMbjCzPx53NLMJZnavpG+kMxoAAACq0WAL6vGSzpD0mJkdbGan\nSnpG0hhJh6U1HAAAAKrPoAqqu/9KSRF9RskV+z+WtFTSTHd/Pb3xAAAAUG2GcpngVEmHS3pTUpek\nAyWNTGMoAAAAVK9BFVQzu1jSryX9h6RDJR0pqVnS02Z2dHrjAQAAoNoMdg/qBZI+6+7nuftmd39G\nSUm9R9KDaQ0HAACA6jOUj5la3XeBu3dKutDMfjbUocxsuaSnldwE4FxJHZK+7e5XDjUbAAAAcRvs\nRVKrt/N3Dw1+nH7OlrReyZ7ZxZKuMLMTU8oGAABApGK+l9rT7n6Vu7/i7rdLelwSBRUAAGAnl/6N\n69Pz9ICv35G02/Y2WLRokZqamvotmzdvnubNm5fyaAAAANiWfD6vZcuW9Vu2du3aorePuaB2Dvja\ntYM9vkuWLFFzc3O4iQAAALBDuVxOuVyu37LW1lYdfvjhRW0f8yF+AAAAVCEKKgAAAKISa0H1rAcA\nAABANqI8B9XdT9jKsjOymAUAAADlFeseVAAAAFQpCioAAACiQkEFAABAVCioAAAAiAoFFQAAAFGJ\n8ir+wdqyZYs2b96ceu6RRx6ZemavN998M1j2u+++Gyx76tSpwbInTJgQLPv5558Plj1+/Phg2SGF\nfH4/+eSTwbIPPfTQYNkhXztdXV3BshsaGoJl77nnnsGy33jjjWDZkyZNCpa9atWqYNmjR48Olt3e\n3h4sO+TjvXr16mDZ7uE+3TLk6/K5554Llr1ly5bUM1988cWi12UPKgAAAKJCQQUAAEBUKKgAAACI\nCgUVAAAAUaGgAgAAICoUVAAAAESFggoAAICoRFtQzazezG4ys/fMbJOZPWxmh2c9FwAAAMKKtqBK\nukHSGZK+IKlZ0suS/t3MxmY6FQAAAIKKsqCa2UhJX5G0yN3vc/fnJf2VpE2SvpTpcAAAAAgqyoIq\naT8lt2H9Ve8Cd++S9Kikg7IaCgAAAOHVZj1Ami655BI1NTX1WzZ79mzNnj07o4kAAACqz89//nP9\n4he/6Lds/fr1RW8fa0F9RVKnpGMkLZMkM6uVdISkG7e10bXXXqvDDjusLAMCAABg60455RSdcsop\n/ZY999xzOuuss4raPsqC6u4bzeyfJd1gZmskvSFpsaQGSd/PdDgAAAAEFWVBLbhYkkm6TVKjpMcl\nnezubZlOBQAAgKCiLajuvkXS/yj8AQAAQJWI9Sp+AAAAVCkKKgAAAKJCQQUAAEBUKKgAAACICgUV\nAAAAUaGgAgAAICrRfszUYNTV1am+vj713K6urtQze+29994VmR1ST09PsOyPfvSjwbIrlbsHyw55\nZ7eQz5OJEycGy25vbw+W3djYGCw75Pvg2LFjg2WHtMceewTL7uzsDJYd8nmyadOmYNkTJkwIlh3S\nww8/HCx7xowZwbJDKOV9mz2oAAAAiAoFFQAAAFGhoAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUK\nKgAAAKIS5eegmtlySc8UvvyCpE5J/+zuV2Q3FQAAAMoh5j2oZysppkdIOl/SQjP7UrYjAQAAILQo\n96AWvOHuCwv//ZKZfVzSAknfy3AmAAAABBbzHtT/GvD1ryUdYGaWxTAAAAAoj5j3oJZs8eLFampq\n6rdszpw5mjdvXkYTAQAAVJ98Pq98Pt9vWVtbW9Hbx1xQjxrw9dGSXnJ339YG119/vZqbm8NOBQAA\ngO3K5XLK5XL9lrW2tqqlpaWo7WM+xL+PmS0xs6lmlpP0VUn/lPVQAAAACCvmPai3SWqQ9KikLklL\n3f2WbEcCAABAaDEX1M7CVfx/l/UgAAAAKJ+YD/EDAACgCsVaULd5IRQAAAB2blEe4nf3E7KeAQAA\nANmIdQ8qAAAAqhQFFQAAAFGhoAIAACAqUZ6DOlj19fUaPnx46rlmlnomtm3YMH5vKqdKfX5X6vOk\nsbEx6xEGpbY23I+LSn1MQqqrq8t6hEFpaGjIeoTozJgxI+sRKlJlvsMDAABgp0VBBQAAQFQoqAAA\nAIgKBRUAAABRoaACAAAgKhRUAAAARIWCCgAAgKhEWVDNbKSZ3WZm7Wb2lpktNLPlZnZj1rMBAAAg\nrCgLqqQlkmZIOl3SyZJmSmrOciAAAACUR3R3kjKzUZL+UtJZ7v5gYdk5kt7Mci4AAACUR4x7UPeT\nVCfp0d4F7r5G0guZTQQAAICyiW4P6lAsWLBAY8eO7bds/vz5yuVyGU0EAABQffL5vPL5fL9lbW1t\nRW9v7p72TENSOMT/gZJD/D8qLBsn6Q1J33H3hVvZZrqkJx5//HFNnz49xEypZwIAAFST1tZWtbS0\nSFKLu7dub93o9qC6+wYz+56kG8zsA0mrJF0tqTvbyQAAAFAO0RXUggsljZL0U0ntkr4pqSnTiQAA\nAFAWMV4kJXff4O7nuHuju+/p7t/MeiYAAACUR5QFFQAAANWrkgpqXFdzAQAAIIhYz0H9EHc/IesZ\nAAAAEF4l7UEFAABAFaCgAgAAICoUVAAAAESlYs5BLUZXV5c6OztTz33//fdTz+w1YsSIisweNizc\n7zZvvfVWsOyRI0cGy25oaAiW3djYGCx7y5YtwbJDPiYor7fffjtYdnt7e7DsAw88MFh2yNfO8OHD\ng2WHnPvNN98Mlj1lypRg2SHvqhny5+XOjEcNAAAAUaGgAgAAICoUVAAAAESFggoAAICoUFABAAAQ\nFQoqAAAAolJxBdXM6rKeAQAAAOFE/zmoZrZc0jOSuiR9XtLTkk7MdCgAAAAEE31BLThb0j9L+kTW\ngwAAACCsSimoL7n7xVkPAQAAgPAq5RzUJ7IeAAAAAOVRKXtQNxSz0qJFi9TU1NRv2bx58zRv3rwg\nQwEAAODD8vm88vl8v2VtbW1Fb18pBbUoS5YsUXNzc9ZjAAAAVLVcLqdcLtdvWWtrq1paWoravlIO\n8QMAAKBKVEJB9awHAAAAQPlEf4jf3U/IegYAAACUTyXsQQUAAEAVoaACAAAgKhRUAAAARIWCCgAA\ngKhQUAEAABAVCioAAACiEv3HTJVi/fr1WrduXeq5w4aF6/FjxowJll1TUxMsu6OjI1j2uHHjKjK7\nq6srWHbI5+B7770XLHvfffcNlt3e3h4su6GhIVh2T09PsOzu7u5g2bvsskuw7AkTJgTLDink+2DI\n7JDP70rlXpkfub5ly5Zg2cOHDw+WXQz2oAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUKKgAAAKJC\nQQUAAEBUKKgAAACIStQF1cxmm9nTZrbRzFab2X1mxge4AQAA7MSi/aB+M5so6U5JiyT9RFKjpBmS\nLMu5AAAAEFa0BVXSHpJqJP3Y3d8oLPtdhvMAAACgDGI+xP+UpP+U9IyZ3WVm55rZ2KyHAgAAQFjR\n7kF19x5JJ5vZ0ZJOlnSepKvN7Ch3f31r21x++eUfurf9rFmzNGvWrODzAgAAIJHP55XP5/sta2tr\nK3r7aAtqL3f/taRfm9lVkl6XdIakf9rauldddZWmTZtWzvEAAAAwQC6XUy6X67estbVVLS0tRW0f\nbUE1syMlnSjpPkl/kPTfJE2Q9GyWcwEAACCsaAuqpHWSPinpAkljlOw9Xeju92U6FQAAAIKKtqC6\n+/OSTsl6DgAAAJRXzFfxAwAAoApRUAEAABAVCioAAACiQkEFAABAVCioAAAAiAoFFQAAAFGJ9mOm\nBqO9vV1r1qxJPbe2NtzDNGxYuN8RSrmlWKmampqCZbt7sOzNmzcHyx4xYkSw7C1btgTLDvkcDKmx\nsTFYdnd3d7Ds+vr6YNnvvvtusOxRo0YFy66rqwuWHVLInw3vv/9+sOyGhoZg2T09PcGyzSxYdk1N\nTbDsF154IVj2/vvvHyw7a5X5kwkAAAA7LQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRib6gmtly\nM7sx6zkAAABQHtEXVAAAAFSXqAuqmd0q6ThJF5hZj5l1m9k+Wc8FAACAcGL/oP4LJE2V9FtJl0sy\nSasynQgAAABBRV1Q3X2dmXVI2ujuFFMAAIAqEPUhfgAAAFSfqPegluqaa6750P25TzvtNJ122mkZ\nTQQAAFB98vm88vl8v2VtbW1Fb18JBbVDUk0xK1566aU65JBDAo8DAACA7cnlcsrlcv2Wtba2qqWl\npajtK+EQ/2uSjjKzyWY23sws64EAAAAQTiUU1CWSuiU9K+kPkiZlOw4AAABCiv4Qv7u/JOmYrOcA\nAABAeVTCHlQAAABUEQoqAAAAokJBBQAAQFQoqAAAAIgKBRUAAABRoaACAAAgKtF/zFQp9t57b+23\n336p57p76pm9Vq5cGSx70qRwHxnb3d0dLLu+vj5Y9pYtW4Jlb968OVh2yMck5PPknnvuCZZ9+umn\nB8sO+Zrv6uoKlr377rsHy3711VeDZe+5557BskPe26WjoyNY9m677RYsO6QDDjgg6xGic+CBB2Y9\nwqBs2rQp9cxSfk6yBxUAAABRoaACAAAgKhRUAAAARIWCCgAAgKhQUAEAABAVCioAAACiQkEFAABA\nVKIsqGb2qpmdP2DZCjO7IquZAAAAUB5RFlQAAABULwoqAAAAokJBBQAAQFRqsx5gG3okDbyBct2O\nNlq4cKGampr6LZs/f75yuVyKowEAAGB77rrrLt111139lrW1tRW9fawFdZWkPXq/MLMxkj6yo41u\nvPFGTZ8+PeRcAAAA2IG5c+dq7ty5/ZatWLFCxxxzTFHbx3qI/wFJXzCzY83sY5L+RVJXtiMBAACg\nHGLdg3qtpH0l3SupTdLlha8BAACwk4uyoLp7u6SzBiy+PYtZAAAAUF6xHuIHAABAlaKgAgAAICoU\nVAAAAESFggoAAICoUFABAAAQFQoqAAAAohLlx0zFpru7O1j2fvvtFyy7UoV8vEeMGBEsu1J1dnYG\ny541a1aw7JDcPVi22cC7OKcn5Nx77bVXsOzhw4cHyw75mFTq8wTl1dUV7j5DtbXhalxDQ0PqmaX8\nDGYPKgAAAKJCQQUAAEBUKKgAAACICgUVAAAAUaGgAgAAICoUVAAAAESFggoAAICoRFlQzeyvzOyt\nrSz/NzO7JYuZAAAAUB5RFlRJd0vaxcyO711gZuMkfUrSHZlNBQAAgOCiLKjuvlbSLySd1WfxHEmr\n3P3BTIYCAABAWURZUAt+IOlMM6srfH2WpGUZzgMAAIAyCHcT16G7V0mBPtXMHpc0Q9IF29tg4cKF\nampq6rds/vz5yuVywYYEAABAf/l8Xvl8vt+ytra2orc3d097ptSY2fcljZH0G0l/4e6HbGO96ZKe\neOyxxzR9+vTU5+jq6ko9s1d9fX2w7ErV3d0dLLumpiZYdqXq7OwMll1XV7fjlSIU8n3RzIJlh5y7\no6MjWPbw4cODZYdUqc8TlFfIDlFbG/N+xg9rbW1VS0uLJLW4e+v21o39/9kPJP1M0iGSbs94FgAA\nAJRBzOegStIDkj6QdICkOzOeBQAAAGUQ9R5UT46f7JX1HAAAACif2PegAgAAoMpQUAEAABAVCioA\nAACiQkEFAABAVCioAAAAiAoFFQAAAFGJ+mOmSrVhwwatW7cu9dwxY8aknolt4w4q5VWpd3sKqVKf\ngyHnDnnHsUq9k1Sl3hUspEp97YRUaXd7igV7UAEAABAVCioAAACiQkEFAABAVCioAAAAiAoFFQAA\nAFGhoAIAACAqFFQAAABEJZqCamanmtkaK3yImplNM7MeM7umzzq3mNlt2U0JAACA0KIpqJIeljRa\nUnPh6+MkrZI0s886n5S0vLxjAQAAoJyiKajuvk7SU/pTIZ0paamkZjMbaWZ7Sdpf0v/LZEAAAACU\nRTQFteAh/amgzpB0j6TnJB2rZO/pW+7+SjajAQAAoBxiu0Hsg5K+aGbTJHW4+4tm9pCk4yWNU1Jg\nt+nSSy/VmDFj+i0788wzNXv27EDjAgAAYKB8Pq98Pt9vWVtbW9Hbm7unPdOgmdlYSasl3SGp3t3P\nMrPPSLpY0lhJ33T3W7ay3XRJTzz44IOaNm1a6nMNLL1pGjYstp3Y2evp6QmWzeMNDM769euDZY8e\nPTpYdqWK6WdzKQrXOQNb1draqpaWFklqcffW7a0b1U9rd18r6WlJn1OyN1VKzjmdLmmqdrAHFQAA\nAJUvqoJa8JCSuR6UJHdfI+lZSe+4+0sZzgUAAIAyiK6guvsCd69x9xf7LGt2972znAsAAADlEV1B\nBQAAQHWjoAIAACAqFFQAAABEhYIKAACAqFBQAQAAEBUKKgAAAKIS261Oh6SxsVFjx47NegwA2Olw\nt6fy4o5MO4+Ojo5g2fX19cGys8YeVAAAAESFggoAAICoUFABAAAQFQoqAAAAokJBBQAAQFQoqAAA\nAIgKBRUAAABRibKgmtkEM3vHzC7us+wTZrbFzI7PcjYAAACEFeUH9bv7ajP7S0k/MbP7JL0o6TZJ\nN7n78mynAwAAQEhRFlRJcvefm9l3JN0p6XFJ6yVdmu1UAAAACC3KQ/x9XKikRM+WdJa7d2Y8DwAA\nAAKLdg9qwf6S9lRSpD8i6dntrbxgwYL/3979hdZ91nEcf3+bZcmiLJlW26o4J8OLuatTi0U3ypT5\nD9yNXuRsXYd/oHqxaSmKyG52M3CXXk5mrX+aBhnM7UJk/qvgTVmSIUI7hP0TlM0Gia5dTJt8vTgJ\nnKRt+ofz/M6T5P2C0J7nPL8v35ZfzvnwnN/5PYyOjq4aa7fbtNvtYg1KkiRptYmJCSYmJlaNzc3N\nXfXxkZm97qknImIQOAnMAC8Bh4A7M/PMJea2gKmpqSlarVazjarnlpaWitXetq32Dw0kSZvJwsJC\nsdo33nhjsdolTE9Ps3v3boDdmTm93tya360fB24GHgaeoBNSj/S1I0mSJBVXZUCNiH3AI8D+zDyb\nnWXeA8BdEXGwv91JkiSppCqvQc3ME8DQmrHXgFv605EkSZKaUuUKqiRJkrYuA6okSZKqYkCVJElS\nVQyokiRJqooBVZIkSVUxoEqSJKkqVd5m6notLi6yuLjY87oXLlzoec0V586dK1Z7bGysWO1adyDb\nrNxd62Ilz8HZ2dlitYeGhq486TqNjIwUq71Rz5OIKFZ7fn6+WO3h4eFitUu8T64YGBgoVrvk73zJ\n82RwcLBY7ZJKvO9cS82N+YojSZKkTcuAKkmSpKoYUCVJklQVA6okSZKqYkCVJElSVQyokiRJqooB\nVZIkSVWpMqBGxIMRcSYiBteMPxMRR/vVlyRJksqrMqACv6TT230rAxHxHuALwFP9akqSJEnlVRlQ\nM3MemAC+0jX8IPBaZv6pP11JkiSpCVUG1GU/Aj4TEbuWHz8EHOljP5IkSWrADf1u4HIy88WI+Atw\nICKeB+4A1r3+9PDhw4yOjq4aGx8fZ3x8vFyjkiRJWmViYoLjx4+vGpubm7vq4yMze91Tz0TEN4Bv\nA88Dt2fm5y8zrwVMnTx5klar1fM+Lly40POaK86dO1es9tjYWLHaNZ8369m2reYPDS5vaWmpWO2N\n+n9S8hycnZ0tVntoaKhY7ZGRkWK1N+p5EhHFas/PzxerPTw8XKz24uJisdoDAwPFapf8nS95nmzU\nvku870xPT7Nnzx6A3Zk5vd7c2l9xjgEfAL6OX46SJEnaEqoOqJn5H+Bp4C3gV31uR5IkSQ2oOqAu\nez/w88w83+9GJEmSVF61X5KKiDHgHmAf8M0+tyNJkqSGVBtQgRlgDPhuZv6t381IkiSpGdUG1My8\nrQRWyMgAAARiSURBVN89SJIkqXkb4RpUSZIkbSEGVEmSJFXFgCpJkqSqVHsN6vUYGBgosotFyZ0x\nSu4qU1LJ3St0sY26i09JJc/B7du3F6utzaPkbk8ludtTszZq3yV20byWXcx815MkSVJVDKiSJEmq\nigFVkiRJVTGgSpIkqSoGVEmSJFXFgCpJkqSqGFAlSZJUlWoDakTcGhFLEbG4/OfKz+/73ZskSZLK\nqflG/a8DO7se7wJ+C5zoTzuSJElqQrUBNTvbUbwJEBFDwLPAnzPzsb42JkmSpKKq/Yh/jSPAO4AH\n+t2IJEmSyqp2BXVFRDwK3Avsycyz6809dOgQo6Ojq8ba7Tbtdrtgh5IkSeo2OTnJ5OTkqrG5ubmr\nPj46n6TXKSK+BPwC+Fxm/nGdeS1gampqilar1VR7kiSpS8lMERHFautiCwsLPa85MzPD3r17AXZn\n5vR6c6tdQY2IjwJHgR8ApyJix/JTC5n57/51JkmSpJJqvgb1Y8BNwKPAP7p+nu5nU5IkSSqr2hXU\nzDxKZwVVkiRJW0jNK6iSJEnaggyokiRJqooBVZIkSVUxoEqSJKkqBlRJkiRVpdpv8V+jYYBTp071\nuw9JkrYsb9S/eZw/f77nNU+fPr3y1+Erza16J6mrFRH309lxSpIkSXV7IDOPrTdhswTUdwOfBV4F\n5vvbjSRJki5hGPgQ8JvMnF1v4qYIqJIkSdo8/JKUJEmSqmJAlSRJUlUMqJIkSaqKAVWSJElVMaBK\nkiSpKgZUSapMRJyIiPErzDkYEc821ZMkNcmAKkkViYj7gPdm5vGusaXl8W4/BloR8clGG5SkBhhQ\nJakuDwNHrjQpM88Dx4BvFe9IkhpmQJWkhkTE9oj4Z0R8r2vsExHxv4i4JyK2A58Cnut6/hUggWeW\nV1Jf7ir5HPDFiBhq6t8gSU0woEpSQzLzDPBV4LGIaEXEO4GfAj/MzD8AdwNnM/NU12F7gAAeAnYu\nP17xAjAIfLyJ/iWpKTf0uwFJ2koy89cR8SSdj+dfAN4Cvr/89AeBN9bMPxMRAHOZ+eaa596OiDng\n1uKNS1KDXEGVpOZ9h84CwZeB+5evJwW4CZi/xlpvAyM97E2S+s6AKknNux14H53X4Nu6xs8At1xj\nrXcB/+pRX5JUBT/il6QGRcQg8DPgOPAS8FRE3Ll8feoMsDMiRjNzruuw88DAJWp9GBhaPk6SNg1X\nUCWpWY8DN9O5ndQTdELqym2lZuisoq69t+mrwKcjYkdEjHWN3w28nJmvFO1YkhpmQJWkhkTEPuAR\nYH9mns3MBA4Ad0XEwcxcAn4C7F9z6GHgXuB1YLprvA08WbxxSWpYdF4fJUk1iIgdwF+BVmb+fZ15\ndwC/Az6Smf9tqj9JaoIrqJJUkcx8A/ganVtOrWcXcMBwKmkzcgVVkiRJVXEFVZIkSVUxoEqSJKkq\nBlRJkiRVxYAqSZKkqhhQJUmSVBUDqiRJkqpiQJUkSVJVDKiSJEmqigFVkiRJVfk/VT2F5g+tOKoA\nAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x11011c7d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "%matplotlib inline\n",
    "import matplotlib.pylab as plt\n",
    "\n",
    "# Reduction table\n",
    "my2ascii_table = {\n",
    "    ord(u'â'):\"a\",\n",
    "    ord(u'ä'):\"e\",\n",
    "    ord(u\"à\"):\"a\",\n",
    "    ord(u\"æ\"):\"a\",\n",
    "    ord(u'ç'):\"c\",\n",
    "    ord(u\"é\"):\"e\",\n",
    "    ord(u\"è\"):\"e\",\n",
    "    ord(u\"ê\"):\"e\",\n",
    "    ord(u\"ë\"):\"e\",\n",
    "    ord(u'ğ'):\"g\",\n",
    "    ord(u'ı'):\"i\",\n",
    "    ord(u\"î\"):\"i\",\n",
    "    ord(u'ï'):\"i\",\n",
    "    ord(u'œ'):\"o\",\n",
    "    ord(u\"ô\"):\"o\",\n",
    "    ord(u'ö'):\"o\",\n",
    "    ord(u'ş'):\"s\",\n",
    "    ord(u'ù'):\"u\",\n",
    "    ord(u\"û\"):\"u\",\n",
    "    ord(u'ü'):\"u\",\n",
    "    ord(u'ß'):\"s\"\n",
    "    }\n",
    "\n",
    "\n",
    "def letter2idx(x):\n",
    "    if x=='_':\n",
    "        i = 0\n",
    "    else:\n",
    "        i = ord(x)-ord('a')+1\n",
    "    \n",
    "    if i<0 or i>26:\n",
    "        i = ord(my2ascii_table[ord(x)])-ord('a')+1\n",
    "        \n",
    "    return i\n",
    "\n",
    "T = np.zeros((27,27))\n",
    "# Convert bigrams to a transition matrix\n",
    "for pair in bigrams:\n",
    "    c = pair[1]\n",
    "    s = list(pair[0])\n",
    "    j = letter2idx(s[0])\n",
    "    i = letter2idx(s[1])\n",
    "    T[i,j] += c\n",
    "\n",
    "plt.figure(figsize=(8,8))\n",
    "\n",
    "alphabet=[chr(i+ord('a')) for i in range(26) ]\n",
    "alphabet.insert(0,'_')\n",
    "M = len(alphabet)\n",
    "\n",
    "plt.imshow(T/np.sum(T,axis=0), interpolation='nearest', vmin=0,cmap='gray_r')\n",
    "plt.xticks(range(M), alphabet)\n",
    "plt.xlabel('x(t)')\n",
    "plt.yticks(range(M), alphabet)\n",
    "plt.ylabel('x(t-1)')\n",
    "ax = plt.gca()\n",
    "ax.xaxis.tick_top()\n",
    "#ax.set_title(f, va='bottom')\n",
    "plt.xlabel('x(t)')\n",
    "\n",
    "plt.show()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Monte Carlo with Spark"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Pi is roughly 3.142848\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pyspark\n",
    "\n",
    "def sample(p):\n",
    "    x, y = 2*np.random.rand()-1, 2*np.random.rand()-1\n",
    "    return 1 if x*x + y*y < 1 else 0\n",
    "\n",
    "NUM_SAMPLES = 1000000\n",
    "\n",
    "count = sc.parallelize(xrange(0, NUM_SAMPLES)).map(sample).reduce(lambda a, b: a + b)\n",
    "\n",
    "print(\"Pi is roughly %f\" % (4.0 * count / NUM_SAMPLES))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Spark application: \n",
    "* a driver program that runs the user’s main function, that executes various parallel operations on a cluster. \n",
    "\n",
    "Main abstraction: \n",
    "* resilient distributed dataset (RDD), \n",
    "* a collection of elements partitioned across the nodes of the cluster that can be operated on in parallel. \n",
    "\n",
    "RDD: \n",
    "* Created from a file in the Hadoop file system (or any other Hadoop-supported file system)\n",
    "* Automatically recover from node failures.\n",
    "\n",
    "Cached: \n",
    "* Ask Spark to persist an RDD in memory\n",
    "\n",
    "Shared variables: \n",
    "* broadcast variables: cache a value in memory on all nodes, \n",
    "* accumulators: counters and sums.\n",
    "\n",
    "\n",
    "\n",
    "Spark’s Python API data formats (as of Nov 2016):\n",
    "\n",
    "* SparkContext.wholeTextFiles:\n",
    "read a directory containing of multiple small text files, return (filename, content) pairs. \n",
    "\n",
    "* RDD.saveAsPickleFile and SparkContext.pickleFile:\n",
    "save an RDD as pickled Python objects. \n",
    "\n",
    "* SequenceFile and Hadoop Input/Output Formats\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading Json"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+-------+--------------------+\n",
      "|   name|          properties|\n",
      "+-------+--------------------+\n",
      "|Product|[[Product identif...|\n",
      "+-------+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from pyspark.sql import SparkSession\n",
    "\n",
    "#    .config(\"spark.some.config.option\", \"some-value\") \\\n",
    "\n",
    "spark = SparkSession \\\n",
    "    .builder \\\n",
    "    .appName(\"Python Spark SQL basic example\") \\\n",
    "    .getOrCreate()\n",
    "\n",
    "df = spark.read.json(\"data/products.json\")\n",
    "# Displays the content of the DataFrame to stdout\n",
    "df.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "+--------------------+\n",
      "|          properties|\n",
      "+--------------------+\n",
      "|[[Product identif...|\n",
      "+--------------------+\n",
      "\n"
     ]
    }
   ],
   "source": [
    "#df.printSchema()\n",
    "df.select(\"properties\").show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Reading Parquet"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "df = spark.read.parquet(\"/Users/cemgil/Downloads/sahibinden_NativeAdFeaturesEdr/dt=20171028/000000_0\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "DataFrame[summary: string, flakeid: string, deliveryid: string, adid: string, version: string, channel: string, bidpricekurus: string, dailybudgetkurus: string, matchedadcity: string, adimpressioncount: string, adclickcount: string, adctr: string, adtitle: string, addescription: string, adcalltoaction: string, accountadcount: string, accountdayssincefirstadcreation: string, eventtype: string, vieweruniqueuserid: string, viewercity: string, vieweradimpressioncount: string, viewertotalimpressioncount: string, viewertotalclickcount: string, viewerip: string, trackingsinceseconds: string, vieweruserid: string, edrtypename: string, edrgroup: string, edrhostname: string, edruniqueid: string]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.describe()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "PySpark (Spark 2.0.0)",
   "language": "python",
   "name": "pyspark"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.13"
  },
  "toc": {
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": "block",
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}