product-recommendation-system.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 536,
   "metadata": {},
   "outputs": [],
   "source": [
    "sc.stop()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize Spark instance"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 537,
   "metadata": {},
   "outputs": [],
   "source": [
    "import findspark\n",
    "\n",
    "findspark.init()\n",
    "\n",
    "import pyspark\n",
    "from pyspark import SparkConf, SparkContext\n",
    "\n",
    "conf = (SparkConf().setMaster(\"spark://Rahuls-MBP:7077\").setAppName(\"prod-reco-system\").set(\"spark.executor.memory\", \"8g\").set(\"spark.driver.memory\",\"3g\").set(\"spark.driver.maxResultSize\",\"0\").set(\"spark.storage.memoryFraction\", \"0.5\"))\n",
    "\n",
    "sc = SparkContext(conf = conf)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 538,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "        <div>\n",
       "            <p><b>SparkContext</b></p>\n",
       "\n",
       "            <p><a href=\"http://rahuls-mbp:4040\">Spark UI</a></p>\n",
       "\n",
       "            <dl>\n",
       "              <dt>Version</dt>\n",
       "                <dd><code>v2.3.0</code></dd>\n",
       "              <dt>Master</dt>\n",
       "                <dd><code>spark://Rahuls-MBP:7077</code></dd>\n",
       "              <dt>AppName</dt>\n",
       "                <dd><code>prod-reco-system</code></dd>\n",
       "            </dl>\n",
       "        </div>\n",
       "        "
      ],
      "text/plain": [
       "<SparkContext master=spark://Rahuls-MBP:7077 appName=prod-reco-system>"
      ]
     },
     "execution_count": 538,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sc"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 342,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "from pyspark.sql import SQLContext\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Initialize data and paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 542,
   "metadata": {},
   "outputs": [],
   "source": [
    "# category=\"Amazon_Instant_Video\"\n",
    "# category='Musical_Instruments'\n",
    "# category='Apps_for_Android'\n",
    "# category='Toys_and_Games'\n",
    "# category='Cell_Phones_and_Accessories'\n",
    "category='Electronics'\n",
    "# category='Books'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 543,
   "metadata": {},
   "outputs": [],
   "source": [
    "#LOCAL\n",
    "datasets_path = os.path.join('..', 'datasets')\n",
    "models_path = os.path.join('..', 'models')\n",
    "\n",
    "datasets_path_local = os.path.join('..', 'datasets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 544,
   "metadata": {},
   "outputs": [],
   "source": [
    "#HDFS\n",
    "datasets_path = \"hdfs://192.168.1.6:9000/data/\"\n",
    "models_path = \"hdfs://192.168.1.6:9000/models/\"\n",
    "\n",
    "datasets_path_local = os.path.join('..', 'datasets')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 545,
   "metadata": {},
   "outputs": [],
   "source": [
    "meta_data_file = os.path.join(datasets_path, 'meta_'+category+'.json')\n",
    "meta_data_file_encoded = os.path.join(datasets_path, 'meta_'+category+'_encoded.csv')\n",
    "\n",
    "meta_data_file_local = os.path.join(datasets_path_local, 'meta_'+category+'.json')\n",
    "meta_data_file_encoded_local = os.path.join(datasets_path_local, 'meta_'+category+'_encoded.csv')\n",
    "\n",
    "ratings_file = os.path.join(datasets_path, 'ratings_'+category+'.csv')\n",
    "ratings_file_encoded = os.path.join(datasets_path, 'ratings_'+category+'_encoded.csv')\n",
    "ratings_file_local = os.path.join(datasets_path_local, 'ratings_'+category+'.csv')\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 546,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'hdfs://192.168.1.6:9000/data/ratings_Electronics_encoded.csv'"
      ]
     },
     "execution_count": 546,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_file_encoded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/05/28 02:17:54 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "18/05/28 02:18:08 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "18/05/28 02:18:14 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Found 25 items\n",
      "-rw-r--r--   1 rahul supergroup  916259348 2018-05-28 02:18 /data/ratings_Books.csv\n",
      "-rw-r--r--   1 rahul supergroup 2540618615 2018-05-28 02:18 /data/meta_Books.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-26 20:26 /data/ratings_Musical_Instruments_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   13329224 2018-05-26 20:25 /data/meta_Musical_Instruments_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   19875726 2018-05-26 20:24 /data/ratings_Musical_Instruments.csv\n",
      "-rw-r--r--   1 rahul supergroup  102040366 2018-05-26 20:24 /data/meta_Musical_Instruments.json\n",
      "-rw-r--r--   1 rahul supergroup  306860900 2018-05-26 18:08 /data/meta_Books_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-26 16:31 /data/ratings_Electronics_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   90331441 2018-05-26 16:21 /data/meta_Electronics_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  318766497 2018-05-26 16:11 /data/ratings_Electronics.csv\n",
      "-rw-r--r--   1 rahul supergroup  663635572 2018-05-26 16:11 /data/meta_Electronics.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-24 01:18 /data/ratings_Toys_and_Games_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   38247253 2018-05-24 01:17 /data/meta_Toys_and_Games_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-23 23:18 /data/ratings_Cell_Phones_and_Accessories_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   63651403 2018-05-23 23:16 /data/meta_Cell_Phones_and_Accessories_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  140470776 2018-05-23 22:58 /data/ratings_Cell_Phones_and_Accessories.csv\n",
      "-rw-r--r--   1 rahul supergroup  406922383 2018-05-23 22:58 /data/meta_Cell_Phones_and_Accessories.json\n",
      "-rw-r--r--   1 rahul supergroup  107498540 2018-05-23 22:53 /data/ratings_Apps_for_Android.csv\n",
      "-rw-r--r--   1 rahul supergroup   59955438 2018-05-23 22:53 /data/meta_Apps_for_Android.json\n",
      "-rw-r--r--   1 rahul supergroup   91781923 2018-05-23 14:24 /data/ratings_Toys_and_Games.csv\n",
      "-rw-r--r--   1 rahul supergroup  470588058 2018-05-23 14:24 /data/meta_Toys_and_Games.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-05 01:06 /data/ratings_Amazon_Instant_Video_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup    1368080 2018-05-05 01:03 /data/meta_Amazon_Instant_Video_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   23791396 2018-05-05 01:03 /data/ratings_Amazon_Instant_Video.csv\n",
      "-rw-r--r--   1 rahul supergroup    2063220 2018-05-05 01:03 /data/meta_Amazon_Instant_Video.json\n"
     ]
    }
   ],
   "source": [
    "# ! hdfs dfs -mkdir /data\n",
    "hdfs_cmd = \"hdfs dfs -put -f \"+meta_data_file_local+\" /data/\"\n",
    "! $hdfs_cmd\n",
    "hdfs_cmd = \"hdfs dfs -put -f \"+ratings_file_local+\" /data/\"\n",
    "! $hdfs_cmd\n",
    "! hdfs dfs -ls -t /data\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process meta.json to meta-encoded.csv"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "hdfs://192.168.1.6:9000/data/meta_Electronics.json\n",
      "Row(prodId=u'0132793040', title=u'Kelby Training DVD: Mastering Blend Modes in Adobe Photoshop CS5 By Corey Barker', categories=[[u'Electronics', u'Computers & Accessories', u'Cables & Accessories', u'Monitor Accessories']])\n",
      "Row(prodIdIndex=93026, title=u'Kelby Training DVD: Mastering Blend Modes in Adobe Photoshop CS5 By Corey Barker', categories=[[u'Electronics', u'Computers & Accessories', u'Cables & Accessories', u'Monitor Accessories']], prodId=u'0132793040')\n"
     ]
    }
   ],
   "source": [
    "sqlContext = SQLContext(sc)\n",
    "\n",
    "print(meta_data_file)\n",
    "df = sqlContext.read.json(meta_data_file)\n",
    "# df.first()\n",
    "\n",
    "df = df.select('asin','title','categories').withColumnRenamed('asin','prodId')\n",
    "\n",
    "print(df.first())\n",
    "\n",
    "\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "\n",
    "prodIdIndexer = StringIndexer(inputCol=\"prodId\", outputCol=\"prodIdIndex\",  handleInvalid='skip')\n",
    "prodIdModel = prodIdIndexer.fit(df)\n",
    "indexedDf = prodIdModel.transform(df)\n",
    "# print(type(indexedDf.select('categories')))\n",
    "# print(indexedDf.first())\n",
    "\n",
    "prodIndexedDf = indexedDf.select('prodIdIndex','title','categories','prodId')\n",
    "\n",
    "from pyspark.sql.types import IntegerType\n",
    "from pyspark.sql.functions import regexp_replace\n",
    "\n",
    "prodIndexedDf = prodIndexedDf.withColumn(\"prodIdIndex\", prodIndexedDf[\"prodIdIndex\"].cast(IntegerType()))\n",
    "prodIndexedDf = prodIndexedDf.withColumn(\"title\", regexp_replace(prodIndexedDf[\"title\"], \"[\\\\r\\\\n]\", \"\"))\n",
    "\n",
    "print(prodIndexedDf.first())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "metadata": {},
   "outputs": [],
   "source": [
    "prodIndexedDf.toPandas().to_csv(meta_data_file_encoded_local, header=True, index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/05/29 19:42:49 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "18/05/29 19:42:53 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Found 26 items\n",
      "-rw-r--r--   1 rahul supergroup   90331441 2018-05-29 19:42 /data/meta_Electronics_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-29 00:13 /data/ratings_Musical_Instruments_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   13329224 2018-05-29 00:12 /data/meta_Musical_Instruments_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-28 03:21 /data/ratings_Books_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  306860900 2018-05-28 02:55 /data/meta_Books_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  916259348 2018-05-28 02:18 /data/ratings_Books.csv\n",
      "-rw-r--r--   1 rahul supergroup 2540618615 2018-05-28 02:18 /data/meta_Books.json\n",
      "-rw-r--r--   1 rahul supergroup   19875726 2018-05-26 20:24 /data/ratings_Musical_Instruments.csv\n",
      "-rw-r--r--   1 rahul supergroup  102040366 2018-05-26 20:24 /data/meta_Musical_Instruments.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-26 16:31 /data/ratings_Electronics_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  318766497 2018-05-26 16:11 /data/ratings_Electronics.csv\n",
      "-rw-r--r--   1 rahul supergroup  663635572 2018-05-26 16:11 /data/meta_Electronics.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-24 01:18 /data/ratings_Toys_and_Games_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   38247253 2018-05-24 01:17 /data/meta_Toys_and_Games_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-23 23:18 /data/ratings_Cell_Phones_and_Accessories_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   63651403 2018-05-23 23:16 /data/meta_Cell_Phones_and_Accessories_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  140470776 2018-05-23 22:58 /data/ratings_Cell_Phones_and_Accessories.csv\n",
      "-rw-r--r--   1 rahul supergroup  406922383 2018-05-23 22:58 /data/meta_Cell_Phones_and_Accessories.json\n",
      "-rw-r--r--   1 rahul supergroup  107498540 2018-05-23 22:53 /data/ratings_Apps_for_Android.csv\n",
      "-rw-r--r--   1 rahul supergroup   59955438 2018-05-23 22:53 /data/meta_Apps_for_Android.json\n",
      "-rw-r--r--   1 rahul supergroup   91781923 2018-05-23 14:24 /data/ratings_Toys_and_Games.csv\n",
      "-rw-r--r--   1 rahul supergroup  470588058 2018-05-23 14:24 /data/meta_Toys_and_Games.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-05 01:06 /data/ratings_Amazon_Instant_Video_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup    1368080 2018-05-05 01:03 /data/meta_Amazon_Instant_Video_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   23791396 2018-05-05 01:03 /data/ratings_Amazon_Instant_Video.csv\n",
      "-rw-r--r--   1 rahul supergroup    2063220 2018-05-05 01:03 /data/meta_Amazon_Instant_Video.json\n"
     ]
    }
   ],
   "source": [
    "hdfs_cmd = \"hdfs dfs -put -f \"+meta_data_file_encoded_local+\" /data/\"\n",
    "! $hdfs_cmd\n",
    "! hdfs dfs -ls -t /data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Process ratings.csv to ratings-encoded.csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Row(userIdIndex=849668, prodIdIndex=93026, rating=u'5.0', userId=u'AKM1MP6P0OYPR', prodId=u'0132793040')\n",
      "6584243\n"
     ]
    }
   ],
   "source": [
    "\n",
    "df = sqlContext.read.csv(ratings_file)\n",
    "# print(df.first())\n",
    "\n",
    "df = df.withColumnRenamed('_c0','userId').withColumnRenamed('_c1','prodId').withColumnRenamed('_c2','rating').withColumnRenamed('_c3','timestamp')\n",
    "\n",
    "from pyspark.ml.feature import StringIndexer\n",
    "\n",
    "prodIdIndexedDf = prodIdModel.transform(df)\n",
    "# print(prodIdIndexedDf.first())\n",
    "\n",
    "useridIndexer = StringIndexer(inputCol=\"userId\", outputCol=\"userIdIndex\", handleInvalid='skip')\n",
    "userIdModel = useridIndexer.fit(prodIdIndexedDf)\n",
    "useridIndexedDf = userIdModel.transform(prodIdIndexedDf)\n",
    "\n",
    "# print(useridIndexedDf.first())\n",
    "\n",
    "useridIndexedDf = useridIndexedDf.withColumn(\"userIdIndex\", useridIndexedDf[\"userIdIndex\"].cast(IntegerType()))\n",
    "useridIndexedDf = useridIndexedDf.withColumn(\"prodIdIndex\", useridIndexedDf[\"prodIdIndex\"].cast(IntegerType()))\n",
    "\n",
    "finalIndexedDf = useridIndexedDf.select('userIdIndex','prodIdIndex','rating','userId','prodId')\n",
    "print(finalIndexedDf.first())\n",
    "print(finalIndexedDf.count())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "metadata": {},
   "outputs": [],
   "source": [
    "finalIndexedDf.write.mode('overwrite').format(\"com.databricks.spark.csv\").option(\"header\", \"true\").save(ratings_file_encoded)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 355,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/05/29 19:49:46 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Found 26 items\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-29 19:49 /data/ratings_Electronics_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   90331441 2018-05-29 19:42 /data/meta_Electronics_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-29 00:13 /data/ratings_Musical_Instruments_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   13329224 2018-05-29 00:12 /data/meta_Musical_Instruments_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-28 03:21 /data/ratings_Books_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  306860900 2018-05-28 02:55 /data/meta_Books_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  916259348 2018-05-28 02:18 /data/ratings_Books.csv\n",
      "-rw-r--r--   1 rahul supergroup 2540618615 2018-05-28 02:18 /data/meta_Books.json\n",
      "-rw-r--r--   1 rahul supergroup   19875726 2018-05-26 20:24 /data/ratings_Musical_Instruments.csv\n",
      "-rw-r--r--   1 rahul supergroup  102040366 2018-05-26 20:24 /data/meta_Musical_Instruments.json\n",
      "-rw-r--r--   1 rahul supergroup  318766497 2018-05-26 16:11 /data/ratings_Electronics.csv\n",
      "-rw-r--r--   1 rahul supergroup  663635572 2018-05-26 16:11 /data/meta_Electronics.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-24 01:18 /data/ratings_Toys_and_Games_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   38247253 2018-05-24 01:17 /data/meta_Toys_and_Games_encoded.csv\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-23 23:18 /data/ratings_Cell_Phones_and_Accessories_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   63651403 2018-05-23 23:16 /data/meta_Cell_Phones_and_Accessories_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup  140470776 2018-05-23 22:58 /data/ratings_Cell_Phones_and_Accessories.csv\n",
      "-rw-r--r--   1 rahul supergroup  406922383 2018-05-23 22:58 /data/meta_Cell_Phones_and_Accessories.json\n",
      "-rw-r--r--   1 rahul supergroup  107498540 2018-05-23 22:53 /data/ratings_Apps_for_Android.csv\n",
      "-rw-r--r--   1 rahul supergroup   59955438 2018-05-23 22:53 /data/meta_Apps_for_Android.json\n",
      "-rw-r--r--   1 rahul supergroup   91781923 2018-05-23 14:24 /data/ratings_Toys_and_Games.csv\n",
      "-rw-r--r--   1 rahul supergroup  470588058 2018-05-23 14:24 /data/meta_Toys_and_Games.json\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-05 01:06 /data/ratings_Amazon_Instant_Video_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup    1368080 2018-05-05 01:03 /data/meta_Amazon_Instant_Video_encoded.csv\n",
      "-rw-r--r--   1 rahul supergroup   23791396 2018-05-05 01:03 /data/ratings_Amazon_Instant_Video.csv\n",
      "-rw-r--r--   1 rahul supergroup    2063220 2018-05-05 01:03 /data/meta_Amazon_Instant_Video.json\n"
     ]
    }
   ],
   "source": [
    "! hdfs dfs -ls -t /data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load CSV data from existing files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 547,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_raw_data = sc.textFile(ratings_file_encoded)\n",
    "ratings_raw_data_header = ratings_raw_data.take(1)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 548,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings_data = ratings_raw_data.filter(lambda line: line!=ratings_raw_data_header).map(lambda line: line.split(\",\")).map(lambda tokens: (int(tokens[0]),int(tokens[1]),float(tokens[2]))).cache()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 549,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6584243\n",
      "[(849668, 93026, 5.0), (263499, 165519, 5.0), (1266909, 86550, 1.0)]\n"
     ]
    }
   ],
   "source": [
    "ratings_count = ratings_data.count()\n",
    "print(ratings_count)\n",
    "print(ratings_data.take(3))\n",
    "# type(ratings_data.first()[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 550,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6584243\n",
      "3697953\n",
      "1.78051019037\n"
     ]
    }
   ],
   "source": [
    "ratings_id_ratings = (ratings_data.map(lambda x: (int(x[0]), float(x[2]))).groupByKey())\n",
    "users_count = ratings_id_ratings.count()\n",
    "ratings_per_user_ratio = float(ratings_count)/users_count \n",
    "print(ratings_count)\n",
    "print(users_count)\n",
    "print(ratings_per_user_ratio)\n",
    "# print(ratings_id_ratings.take(3))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 551,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 5, 1, 3, 1, 3, 1, 5, 1, 1]\n",
      "6584243\n",
      "(1, 769360)\n",
      "(2, 384884)\n",
      "(3, 532423)\n",
      "(4, 1241210)\n",
      "(5, 3656366)\n",
      "6584243\n"
     ]
    }
   ],
   "source": [
    "data_distribution = ratings_data.map(lambda x: int(math.floor(float(x[2]))))\n",
    "print(data_distribution.take(10))\n",
    "print(data_distribution.count())\n",
    "\n",
    "count_dict = dict()\n",
    "count_sum = 0\n",
    "for i in range(1,6):\n",
    "    data_distribution_group = data_distribution.filter(lambda x: x==i)\n",
    "    count_i = data_distribution_group.count()\n",
    "    count_sum += count_i\n",
    "    count_dict[i] = count_i\n",
    "    print(i, count_i)\n",
    "print(count_sum)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 552,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAaEAAAEWCAYAAADPZygPAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3XucXVV99/HPl4RLSgIJMI0hF4Ia\npRFLCgPESytChQSRoI/lQREiTyRaoWKlLZeqiYCKWsXSKi0pKQlaIwWBiMGYAoK2BJhAuISLjAFK\nQiCBhFxAAwm/54+9DuwMM2fOXM6sTM73/Xqd1+y99tpr/fY5Z87v7L3X2VsRgZmZWQ475Q7AzMwa\nl5OQmZll4yRkZmbZOAmZmVk2TkJmZpaNk5CZmWXjJLQDkPQvkr7US22NkbRJ0oA0/0tJn+qNtlN7\nN0ma2lvtdaHfiyQ9J+mZPuov13Z+XdLn+7rfepB0raTJdWx/pqQf1Kv93tKb/9/bIyeh7ZykJyT9\nTtJGSS9I+h9Jn5H02msXEZ+JiAtrbOvPq9WJiP+NiMERsbUXYn/DP3lETI6IOT1tu4txjAHOBsZH\nxJvaWX6EpFdT8t0o6VFJp3Wh/e1lO5uAU4F/TfPl7dokaYWkqyUd2oU2++SDuoN+vgFc1MN2Q9KL\npedgk6S/60mbbdo/QtKK3mqvPbX+f/dXTkL9w4ciYgiwH3AxcA5wRW93Imlgb7e5nRgDPB8Rq6vU\neToiBgN7AH8NzJL09j6Jrvd8ElgQEb8rlVW2awgwEXgE+JWkozLE1yURcRewh6TmHjZ1UPpiVXl8\nszfiq9UO/H/VOyLCj+34ATwB/HmbssOAV4ED0/yVwEVpeh/gRuAFYC3wK4ovG1eldX4HbAL+DhgL\nBDAN+F/g9lLZwNTeL4GvA3cBG4AbgL3SsiOAFe3FC0wCXgZeSf3dV2rvU2l6J+CLwJPAamAusGda\nVoljaortOeDvqzxPe6b116T2vpja//O0za+mOK5sZ932tmM18Bel+X8EnkrPwRLgT1N5Ldv5SeDX\nwD8A64DHgcmltvdPz/1G4L+A7wE/SMt2A34APJ9e07uB4R08B7cAn6i2Xan8n4GWHmzbacDDKd7l\nwKdLbbX7/kvL9gWuTa/R48DnqvWTls0CZvTg/yeAt3awbGbleU7zE4H/SbHfBxxRWrYX8O/A0+k1\nvB7Yvc17a1PaxpnANel12wB8CtgV+G5a/+k0vWv5daLYW18NrAJOK/V9Jen/O81PAZamtn8LTCq9\nz5an1+Vx4OTcn181vUa5A/CjkxeonSSUyv8X+Ms0/dqblCJh/Auwc3r8KaD22uL1D/q56R9qEO0n\noZXAganOtbz+AXkEHSShNL3NP3mpvcqH8/8DWoE3A4OBnwBXtYltVorrIGAz8EcdPE9zKRLkkLTu\nb4BpHcXZZt3XllMkruMpPlj+pFTnE8DewMD0YfEMsFuN2/lJig/Y04EBwF9SfBBVXpc7KBLULsB7\nKT5cKs/xp4GfAn+Q1j0E2KOD7VgDHNredrWpd2Tavt27uW0fBN4CCHgf8BJwcLX3X3pelwBfTtv5\nZooPzGM66ieVfwH4SQ/+f2pKQsBIikR/bIr1A2m+KS3/GfBjYFjarvdV+R+YmV7vE1Jbg4ALgMXA\nHwJNFMnuwlIbW1KdnVMMLwHD2vn/PgxYn+LbKcV9AMX/5gbg7aneCOAduT+/ann4cFz/9TTFt7O2\nXqF4A+4XEa9ExK8ivSurmBkRL8a2h3HKroqIByPiReBLwImVgQs9dDLwnYhYHhGbgPOAk9ocvvhK\nRPwuIu6j+HZ6UNtGUiwnAedFxMaIeAL4NnBKF2LZV9ILFN9srwO+EBH3VhZGxA8i4vmI2BIR36b4\nZtuVw3VPRsSsKM61zaF4jYan81WHAl+OiJcj4tfA/NJ6r1AkiLdGxNaIWBIRGzroYyjFt+DOPE2R\nGIZ2Z9si4mcR8dso3Ab8giLZVOJt7/13KMUH+gVpO5dTfME4qZNYN1bi7IF70vnUyuOYdup8guJQ\n5oKIeDUiFgEtwLGSRgCTgc9ExLq0Xbd10ucdEXF9aut3FO/1CyJidUSsAb7Ctu/PV9LyVyJiAcVe\nVXuvwTRgdkQsSm2vjIhH0rJXgQMlDYqIVRGxrMbnJysnof5rJMXhjra+RbF38QtJyyWdW0NbT3Vh\n+ZMU39b2qSnK6vZN7ZXbHggML5WVR7O9RLHH1NY+Kaa2bY3sQixPR8RQinNCl1LsLbxG0t9IeljS\n+pSs9qRrz8Fr2xERL6XJwRTPwdpSGWz7fF8FLATmSXpa0jcl7dxBH+so9gQ7M5JiD+EF6Pq2SZos\nabGktan+saX6Hb3/9iMl+soDOJ9tX+v2DKnE2U4cN5UGG5xcpY2DI2Jo6bGwnTr7AX/RJr73UiTU\n0RSv0bpOYi1r+z/V3nt939L88xGxpTTf0Xt9NMUhuG2kL4j/F/gMsErSzyQd0IV4s3ES6ofS6KaR\nFOcZtpH2BM6OiDdTHFb6QukkdEd7RJ3tKY0uTY+h+Nb2HPAixWGiSlwDKA411Nru0xT//OW2twDP\ndrJeW8+lmNq2tbKL7RARmykGfrxT0gkAkv6U4hzaiRSHSIZSHBJRZbWu9lOyCthL0h+Uyl57vtM3\n469ExHjg3cBxFCPg2nM/8LYa+vwwcE9EvNjVbZO0K8Uh2X+gODc1FFhQqV/l/fcU8HibZDAkIo5t\nr5+SP6LYA36DKEYgVgYb/LCG7a7mKYo9/nJ8u0fExWnZXpLa2yOr9X+qvff6092M8y3tdhixMCI+\nQJE4H6HY09zuOQn1I5L2kHQcMI/iWPYD7dQ5TtJbJYniw2QrxW46FB/ub+5G15+QND59UF4AXJMO\nK/0G2E3SB9O38y9SHMqpeBYYWx5O3saPgL+WtL+kwcDXgB+3+UbYqRTL1cBXJQ2RtB/FuYRuDS2O\niJcpDud9ORUNoUiOa4CBkr5MscdU0dl2VuvrSYrDPjMl7SLpXcCHKsslvV/SO1OC30CRbF9tvzUW\nUJyjeQMVRkqaQXGi/PxubtsuFK/xGmCLit/xHF3qp6P3313ARknnSBokaYCkA0vDxTt6Dt8H3NTB\n9vamHwAfknRMim23NPx6VESsSjF8X9IwSTtL+rNS3HtL2rOT9n8EfFFSk6R9KN5b3Xl/XgGcJuko\nSTul1/QAScMlTZG0O8W50010/D7ZrjgJ9Q8/lbSR4lvQ3wPfoRih1J5xFCOsNlGc8P5+RNyaln2d\n4h/hBUl/04X+r6I4OfoMxWitzwFExHrgs8C/Uex1vEgxyqfiP9Pf5yXd0067s1Pbt1OM5vk98Fdd\niKvsr1L/yyn2EP8jtd9ds4Exkj5EcTjs5xRJ98kUZ/lwS2fb2ZmTgXdRnAi/iOIE+Oa07E0UI602\nUIxIu43iOWvPXIpzGINKZftKqozcuht4J8Wor1+k5V3atojYSPH6X01x+O/jbHsOq933X/qicBww\ngeK1fo7ifVP58H7Dc5gS1KYohmr3xH3a9ndC321bISKeohh1dj5Fgn0K+Fte/4w8heILwCMUI9g+\nn9Z7hCLBLE//V/vSvosovmzcDzwA3EM3fgOVnovTgEsokvxtFHtYO1F88Xqa4jD9+ygGwGz3KqNz\nzGw7IenHwCMRMaMb634NWB0Rb/ig7W8kXQtckU7U2w7KScgss/SNfy3FHsLRFL9BeVd5dJ7Zjsq/\n5DXL700Uv5Ham+Jw5l86AVmj8J6QmZll44EJZmaWjQ/HdWKfffaJsWPH5g7DzKxfWbJkyXMR0dRZ\nPSehTowdO5aWlpbcYZiZ9SuSnuy8lg/HmZlZRk5CZmaWjZOQmZll4yRkZmbZOAmZmVk2TkJmZpaN\nk5CZmWXjJGRmZtk4CZmZWTa+YoKZWS/S6eq8Uj8Rs+p/gWvvCZmZWTZOQmZmlo2TkJmZZeMkZGZm\n2dQtCUnaTdJdku6TtEzSV1L5lZIel7Q0PSakckm6VFKrpPslHVxqa6qkx9Jjaqn8EEkPpHUulaRU\nvpekRan+IknDOuvDzMz6Xj33hDYDR0bEQcAEYJKkiWnZ30bEhPRYmsomA+PSYzpwGRQJBZgBHA4c\nBsyoJJVU5/TSepNS+bnAzRExDrg5zXfYh5mZ5VG3JBSFTWl25/SoNt5vCjA3rbcYGCppBHAMsCgi\n1kbEOmARRUIbAewREYsjIoC5wAmltuak6Tltytvrw8zMMqjrOSFJAyQtBVZTJJI706KvpsNhl0ja\nNZWNBJ4qrb4ilVUrX9FOOcDwiFiVpp8BhnfSh5mZZVDXJBQRWyNiAjAKOEzSgcB5wAHAocBewDl1\njiGovgf2BpKmS2qR1LJmzZo6RWZmZn0yOi4iXgBuBSZFxKp0OGwz8O8U53kAVgKjS6uNSmXVyke1\nUw7wbOUwW/q7upM+2sZ7eUQ0R0RzU1NTVzfXzMxqVM/RcU2ShqbpQcAHgEdKyUEU52oeTKvMB05N\nI9gmAuvTIbWFwNGShqUBCUcDC9OyDZImprZOBW4otVUZRTe1TXl7fZiZWQb1vHbcCGCOpAEUye7q\niLhR0i2SmgABS4HPpPoLgGOBVuAl4DSAiFgr6ULg7lTvgohYm6Y/C1wJDAJuSg+Ai4GrJU0DngRO\nrNaHmZnloeKUiXWkubk5WlpacodhZv2EL2BakLQkIpo7q+crJpiZWTZOQmZmlo2TkJmZZeMkZGZm\n2TgJmZlZNk5CZmaWjZOQmZll4yRkZmbZOAmZmVk2TkJmZpaNk5CZmWXjJGRmZtk4CZmZWTZOQmZm\nlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaWjZOQmZll4yRkZmbZ1C0JSdpN0l2S7pO0TNJXUvn+ku6U\n1Crpx5J2SeW7pvnWtHxsqa3zUvmjko4plU9KZa2Szi2Vd7kPMzPre/XcE9oMHBkRBwETgEmSJgLf\nAC6JiLcC64Bpqf40YF0qvyTVQ9J44CTgHcAk4PuSBkgaAHwPmAyMBz6W6tLVPszMLI+6JaEobEqz\nO6dHAEcC16TyOcAJaXpKmictP0qSUvm8iNgcEY8DrcBh6dEaEcsj4mVgHjAlrdPVPszMLIO6nhNK\neyxLgdXAIuC3wAsRsSVVWQGMTNMjgacA0vL1wN7l8jbrdFS+dzf6aBv3dEktklrWrFnTvY03M7NO\n1TUJRcTWiJgAjKLYczmgnv31loi4PCKaI6K5qakpdzhmZjusPhkdFxEvALcC7wKGShqYFo0CVqbp\nlcBogLR8T+D5cnmbdToqf74bfZiZWQb1HB3XJGlomh4EfAB4mCIZfTRVmwrckKbnp3nS8lsiIlL5\nSWlk2/7AOOAu4G5gXBoJtwvF4IX5aZ2u9mFmZhkM7LxKt40A5qRRbDsBV0fEjZIeAuZJugi4F7gi\n1b8CuEpSK7CWIqkQEcskXQ08BGwBzoiIrQCSzgQWAgOA2RGxLLV1Tlf6MDOzPOQdgeqam5ujpaUl\ndxhm1k/o9B1nwG3M6n5+kLQkIpo7q+crJpiZWTZOQmZmlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaW\njZOQmZll4yRkZmbZOAmZmVk2TkJmZpaNk5CZmWXTaRKStLukndL02yQdL2nn+odmZmY7ulr2hG4H\ndpM0EvgFcApwZT2DMjOzxlBLElJEvAR8BPh+RPwF8I76hmVmZo2gpiQk6V3AycDPUtmA+oVkZmaN\nopYk9HngPOC6dIO5N1PcudTMzKxHOr2zakTcBtxWml8OfK6eQZmZWWPoNAlJ+inQ9vZ664EW4F8j\n4vf1CMzMzHZ8tRyOWw5sAmalxwZgI/C2NG9mZtYttSShd0fExyPip+nxCeDQiDgDOLijlSSNlnSr\npIckLZN0ViqfKWmlpKXpcWxpnfMktUp6VNIxpfJJqaxV0rml8v0l3ZnKfyxpl1S+a5pvTcvHdtaH\nmZn1vVqS0GBJYyozaXpwmn25ynpbgLMjYjwwEThD0vi07JKImJAeC1K744GTKIZ/TwK+L2mApAHA\n94DJwHjgY6V2vpHaeiuwDpiWyqcB61L5Jaleh33U8ByYmVkd1JKEzgZ+nfZqfgn8CvgbSbsDczpa\nKSJWRcQ9aXoj8DAwsko/U4B5EbE5Ih4HWoHD0qM1IpZHxMvAPGCKJAFHAtek9ecAJ5TaqsR2DXBU\nqt9RH2ZmlkEto+MWSBoHHJCKHi0NRvhuLZ2kw2F/AtwJvAc4U9KpFIMbzo6IdRQJanFptRW8nrSe\nalN+OLA38EJEbGmn/sjKOhGxRdL6VL9aH+V4pwPTAcaMGdN2sZmZ9ZJaL2B6CMUhrIOAE1MCqYmk\nwcC1wOcjYgNwGfAWYAKwCvh2lyLuAxFxeUQ0R0RzU1NT7nDMzHZYtQzRvooiaSwFtqbiAObWsO7O\nFAnohxHxE4CIeLa0fBZwY5pdCYwurT4qldFB+fPAUEkD095QuX6lrRWSBgJ7pvrV+jAzsz7WaRIC\nmoHxEdH2t0JVpXMwVwAPR8R3SuUjImJVmv0w8GCang/8h6TvAPsC44C7AAHjJO1PkTBOAj4eESHp\nVuCjFOeJpgI3lNqaCtyRlt+S6nfUh5mZZVBLEnoQeBPFobOueA/FFbcfkLQ0lZ1PMbptAsXe1BPA\npwHSJYGuBh6iGFl3RkRsBZB0JrCQ4pp1syNiWWrvHGCepIuAeymSHunvVZJagbUUiatqH2Zm1vfU\n2Q5O2tuYQLHHsLlSHhHH1ze07UNzc3O0tLTkDsPM+gmdrtwh9JqY1aUDYNuQtCQimjurV8ue0Mxu\nR2FmZlZFrRcwNTMz63UdJiFJv46I90rayLYXMBUQEbFH3aMzM7MdWodJKCLem/4O6btwzMyskXT6\nY9X0O6FOy8zMzLqqlismvKM8k378eUh9wjEzs0bSYRJKtzzYCPyxpA3psRF4ltd/FGpmZtZtHSah\niPh6Oh/0rYjYIz2GRMTeEXFeH8ZoZmY7qFqGaJ8naRjFJW52K5XfXs/AzMxsx1fLBUw/BZxFcbHP\npRQ3qLuD4l4+ZmZm3VbLwISzgEOBJyPi/RT3BXqhrlGZmVlDqCUJ/b5yEztJu0bEI8Db6xuWmZk1\nglquHbdC0lDgemCRpHXAk/UNy8zMGkEtAxM+nCZnpitq7wn8vK5RmZlZQ6iahCQNAJZFxAHgi5ma\nmVnvqnpOKN3w7VFJY/ooHjMzayC1nBMaBiyTdBfwYqWwUW5qZ2Zm9VNLEvpS3aMwM7OG5JvamZlZ\nNrX8TqhbJI2WdKukhyQtk3RWKt9L0iJJj6W/w1K5JF0qqVXS/ZIOLrU1NdV/TNLUUvkhkh5I61wq\nSd3tw8zM+l7dkhCwBTg7IsZTXOrnDEnjgXOBmyNiHHBzmgeYTHF9unHAdOAyKBIKMAM4HDgMmFFJ\nKqnO6aX1JqXyLvVhZmZ5VLuVw83p7ze603BErIqIe9L0RuBhYCQwBZiTqs0BTkjTU4C5UVgMDJU0\nAjgGWBQRayNiHbAImJSW7RERiyMigLlt2upKH2ZmlkG1c0IjJL0bOF7SPEDlhZUEUwtJYymuOXcn\nMDwiVqVFzwDD0/RI4KnSaitSWbXyFe2U040+VmFmZn2uWhL6MsXIuFHAd9osC2q8irakwcC1wOcj\nYkM6bVM0EhGSoksRd1F3+pA0neJwHWPG+CdSZmb1Uu2mdtdExGTgmxHx/jaPWhPQzhQJ6IcR8ZNU\n/GzlEFj6uzqVrwRGl1YflcqqlY9qp7w7fbTd9ssjojkimpuammrZVDMz64ZOByZExIWSjpf0D+lx\nXC0Np5FqVwAPR0R5T2o+UBnhNpXXbxU+Hzg1jWCbCKxPh9QWAkdLGpYGJBwNLEzLNkiamPo6tU1b\nXenDzMwyqOWmdl+nGJX2w1R0lqR3R8T5naz6HuAU4AFJS1PZ+cDFwNWSplFcjfvEtGwBcCzQCrwE\nnAYQEWslXQjcnepdEBFr0/RngSuBQcBN6UFX+zAzszxUDCyrUkG6H5gQEa+m+QHAvRHxx30QX3bN\nzc3R0tKSOwwz6yd0ujqv1E/ErO6fspe0JCKaO6tX6++Ehpam9+xeSGZmZtuq5dpxXwfuTfcSEvBn\nvP7jTzMzs26r5dpxP5L0S+DQVHRORDxT16jMzKwh1LInRBpBNr/OsZiZWYOp57XjzMzMqnISMjOz\nbKomIUkDJD3SV8GYmVljqZqEImIr8KgkX0DNzMx6XS0DE4YByyTdBbxYKYyI4+sWlZmZNYRaktCX\n6h6FmZk1pFp+J3SbpP2AcRHxX5L+ABhQ/9DMzGxH1+noOEmnA9cA/5qKRgLX1zMoMzNrDLUM0T6D\n4orYGwAi4jHgD+sZlJmZNYZaktDmiHi5MiNpIMWdVc3MzHqkliR0m6TzgUGSPgD8J/DT+oZlZmaN\noJYkdC6wBngA+DTFjeG+WM+gzMysMdQyOu5VSXOAOykOwz0and0Jz8zMrAa13N77g8C/AL+luJ/Q\n/pI+HRE3VV/TzMysulp+rPpt4P0R0Qog6S3AzwAnITMz65FazgltrCSgZDmwsU7xmJlZA+kwCUn6\niKSPAC2SFkj6pKSpFCPj7u6sYUmzJa2W9GCpbKaklZKWpsexpWXnSWqV9KikY0rlk1JZq6RzS+X7\nS7ozlf9Y0i6pfNc035qWj+2sDzMzy6PantCH0mM34FngfcARFCPlBtXQ9pXApHbKL4mICemxAEDS\neOAk4B1pne+n20gMAL4HTAbGAx9LdQG+kdp6K7AOmJbKpwHrUvklqV6HfdSwHWZmVicdnhOKiNN6\n0nBE3F7eC+nEFGBeRGwGHpfUChyWlrVGxHIASfOAKZIeBo4EPp7qzAFmApeltmam8muAf5akKn3c\n0d1tNDOznqlldNz+wF8BY8v1e3ArhzMlnQq0AGdHxDqK69EtLtVZkcoAnmpTfjiwN/BCRGxpp/7I\nyjoRsUXS+lS/Wh/bkDQdmA4wZoxvpWRmVi+1DEy4HngC+CeKkXKVR3dcBrwFmACs6kE7dRURl0dE\nc0Q0NzU15Q7HzGyHVcsQ7d9HxKW90VlEPFuZljQLuDHNrgRGl6qOSmV0UP48MFTSwLQ3VK5faWtF\nus7dnql+tT7MzCyDWvaE/lHSDEnvknRw5dGdziSNKM1+GKiMnJsPnJRGtu0PjAPuohiFNy6NhNuF\nYmDB/HTFhluBj6b1pwI3lNqamqY/CtyS6nfUh5mZZVLLntA7gVMoBgK8msoizXdI0o8oRtPtI2kF\nMAM4QtKEtP4TFNeiIyKWSboaeAjYApwREVtTO2cCCylupDc7IpalLs4B5km6CLgXuCKVXwFclQYe\nrKVIXFX7MDOzPNTZZeDSh/n48u0cGklzc3O0tLTkDsPM+gmdrtwh9JqY1f3LhEpaEhHNndWr5XDc\ng8DQbkdiZmbWgVoOxw0FHpF0N7C5UtiDIdpmZmZAbUloRt2jMDOzhlTL/YRu64tAzMys8dRyxYSN\nFKPZAHYBdgZejIg96hmYmZnt+GrZExpSmS5dg21iPYMyM7PGUMvouNdE4XrAt0EwM7Meq+Vw3EdK\nszsBzcDv6xaRmZk1jFpGx32oNL2F4koHU+oSjZmZNZRazgn16L5CZtZ4dpSrBvTkigFWmw6TkKQv\nV1kvIuLCOsRjZmYNpNqe0IvtlO1OcfvsvQEnITMz65Fqt/d+7YZzkoYAZwGnAfPYTm9GZ2Zm/UvV\nc0KS9gK+AJwMzAEOTrfjNjMz67Fq54S+BXwEuBx4Z0Rs6rOozMysIVT7serZwL7AF4GnJW1Ij42S\nNvRNeGZmtiOrdk6oS1dTMDMz6yonGjMzy8ZJyMzMsqlbEpI0W9JqSQ+WyvaStEjSY+nvsFQuSZdK\napV0v6SDS+tMTfUfkzS1VH6IpAfSOpemK3x3qw8zM8ujnntCVwKT2pSdC9wcEeOAm9M8wGRgXHpM\nBy6D14aIzwAOBw4DZlSSSqpzemm9Sd3pw8zM8qlbEoqI24G1bYqnUPzeiPT3hFL53HSriMXAUEkj\nKG4ZsSgi1qbfJy0CJqVle0TE4ogIYG6btrrSh5mZZdLX54SGR8SqNP0MMDxNjwSeKtVbkcqqla9o\np7w7fbyBpOmSWiS1rFmzpsZNMzOzrso2MCHtwdT1ErXd7SMiLo+I5ohobmpqqkNkZmYGfZ+Enq0c\nAkt/V6fylcDoUr1Rqaxa+ah2yrvTh5mZZdLXSWg+UBnhNhW4oVR+ahrBNhFYnw6pLQSOljQsDUg4\nGliYlm2QNDGNiju1TVtd6cPMzDKp5c6q3SLpR8ARwD6SVlCMcrsYuFrSNOBJ4MRUfQFwLNAKvERx\ntW4iYq2kC4G7U70LIqIy2OGzFCPwBgE3pQdd7cPMzPKpWxKKiI91sOioduoGcEYH7cwGZrdT3gIc\n2E75813tw8zM8vAVE8zMLBsnITMzy8ZJyMzMsnESMjOzbJyEzMwsGychMzPLxknIzMyyqdvvhAx0\nunKH0GtiVl0v82dmDcp7QmZmlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaWjZOQmZll4yRkZmbZOAmZ\nmVk2TkJmZpaNk5CZmWXjJGRmZtlkSUKSnpD0gKSlklpS2V6SFkl6LP0dlsol6VJJrZLul3RwqZ2p\nqf5jkqaWyg9J7bemdVWtDzMzyyPnntD7I2JCRDSn+XOBmyNiHHBzmgeYDIxLj+nAZVAkFGAGcDhw\nGDCjlFQuA04vrTepkz7MzCyD7ekq2lOAI9L0HOCXwDmpfG5EBLBY0lBJI1LdRRGxFkDSImCSpF8C\ne0TE4lQ+FzgBuKlKH2a9yldQN6tNrj2hAH4haYmk6alseESsStPPAMPT9EjgqdK6K1JZtfIV7ZRX\n62MbkqZLapHUsmbNmi5vnJmZ1SbXntB7I2KlpD8EFkl6pLwwIkJSXb9+VesjIi4HLgdobm7210Az\nszrJsicUESvT39XAdRTndJ5Nh9lIf1en6iuB0aXVR6WyauWj2imnSh9mZpZBnychSbtLGlKZBo4G\nHgTmA5URblOBG9L0fODUNEpuIrA+HVJbCBwtaVgakHA0sDAt2yBpYhoVd2qbttrrw8zMMshxOG44\ncF0aNT0Q+I+I+Lmku4GrJU0DngROTPUXAMcCrcBLwGkAEbFW0oXA3aneBZVBCsBngSuBQRQDEm5K\n5Rd30IeZmWXQ50koIpYDB7UF9OvkAAAFbUlEQVRT/jxwVDvlAZzRQVuzgdntlLcAB9bah5mZ5eEr\nJpiZWTbb0++EbAfj38qYWWe8J2RmZtk4CZmZWTZOQmZmlo2TkJmZZeMkZGZm2TgJmZlZNk5CZmaW\njZOQmZll4yRkZmbZOAmZmVk2TkJmZpaNk5CZmWXjJGRmZtk4CZmZWTZOQmZmlo2TkJmZZeMkZGZm\n2TgJmZlZNg2ZhCRNkvSopFZJ5+aOx8ysUTVcEpI0APgeMBkYD3xM0vi8UZmZNaaGS0LAYUBrRCyP\niJeBecCUzDGZmTUkRUTuGPqUpI8CkyLiU2n+FODwiDizVGc6MD3Nvh14tM8D7Zp9gOdyB5FJI287\nNPb2e9u3b/tFRFNnlQb2RST9TURcDlyeO45aSWqJiObcceTQyNsOjb393vYdY9sb8XDcSmB0aX5U\nKjMzsz7WiEnobmCcpP0l7QKcBMzPHJOZWUNquMNxEbFF0pnAQmAAMDsilmUOq6f6zaHDOmjkbYfG\n3n5v+w6g4QYmmJnZ9qMRD8eZmdl2wknIzMyycRLqxyTNlrRa0oO5Y+lrkkZLulXSQ5KWSTord0x9\nRdJuku6SdF/a9q/kjqmvSRog6V5JN+aOpa9JekLSA5KWSmrJHU9P+ZxQPybpz4BNwNyIODB3PH1J\n0ghgRETcI2kIsAQ4ISIeyhxa3UkSsHtEbJK0M/Br4KyIWJw5tD4j6QtAM7BHRByXO56+JOkJoDki\ntvcfq9bEe0L9WETcDqzNHUcOEbEqIu5J0xuBh4GReaPqG1HYlGZ3To+G+TYpaRTwQeDfcsdiPeck\nZP2epLHAnwB35o2k76TDUUuB1cCiiGiYbQe+C/wd8GruQDIJ4BeSlqRLjPVrTkLWr0kaDFwLfD4i\nNuSOp69ExNaImEBxxY/DJDXE4VhJxwGrI2JJ7lgyem9EHExxJ4Az0mH5fstJyPqtdD7kWuCHEfGT\n3PHkEBEvALcCk3LH0kfeAxyfzovMA46U9IO8IfWtiFiZ/q4GrqO4M0C/5SRk/VI6OX8F8HBEfCd3\nPH1JUpOkoWl6EPAB4JG8UfWNiDgvIkZFxFiKS27dEhGfyBxWn5G0exqIg6TdgaOBfj061kmoH5P0\nI+AO4O2SVkialjumPvQe4BSKb8JL0+PY3EH1kRHArZLup7gW4qKIaLihyg1qOPBrSfcBdwE/i4if\nZ46pRzxE28zMsvGekJmZZeMkZGZm2TgJmZlZNk5CZmaWjZOQmZll4yRklpGkrWl4+YOSflr5/U+V\n+kMlfbY0v6+ka+ofqVl9eIi2WUaSNkXE4DQ9B/hNRHy1Sv2xwI2NdtV023F5T8hs+3EH6UrgkgZL\nulnSPeneMVNSnYuBt6S9p29JGlu5n5SkT0r6iaSfS3pM0jcrDUuaJuk36T5EsyT9c59vnVk7BuYO\nwMyKq2IDR1Fcigjg98CHI2KDpH2AxZLmA+cCB6aLl1b2jMomUFxRfDPwqKR/ArYCXwIOBjYCtwD3\n1XWDzGrkJGSW16B0S4aRFPdEWpTKBXwtXSH51bR8eA3t3RwR6wEkPQTsB+wD3BYRa1P5fwJv69Wt\nMOsmH44zy+t3aa9mP4rEc0YqPxloAg5Jy58Fdquhvc2l6a34i6Zt55yEzLYDEfES8DngbEkDgT0p\n7pvziqT3UyQpKA6nDeli83cD75M0LLX9f3orbrOechIy205ExL3A/cDHgB8CzZIeAE4l3aohIp4H\n/jsN6f5Wje2uBL5GcdXl/waeANb3+gaYdYOHaJs1AEmDI2JT2hO6DpgdEdfljsvMe0JmjWFmGgDx\nIPA4cH3meMwA7wmZmVlG3hMyM7NsnITMzCwbJyEzM8vGScjMzLJxEjIzs2z+P+UtEpESE+b2AAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114f6c050>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "plt.bar(list(count_dict.keys()), count_dict.values(), color='darkgreen')\n",
    "plt.xlabel(\"Rating\")\n",
    "plt.ylabel(\"Number of ratings\")\n",
    "plt.title(\"Distribution of Ratings (Dataset) - \"+str(category))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Load product meta_(category)_encoded.csv\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 553,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "427097\n",
      "[(93026, u'Kelby Training DVD: Mastering Blend Modes in Adobe Photoshop CS5 By Corey Barker', '\"[[u\\'Electronics\\''), (165519, u'Kelby Training DVD: Adobe Photoshop CS5 Crash Course By Matt Kloskowski', '\"[[u\\'Electronics\\''), (86550, u'Digital Organizer and Messenger', '\"[[u\\'Electronics\\'')]\n"
     ]
    }
   ],
   "source": [
    "product_meta_data_raw = sc.textFile(meta_data_file_encoded)\n",
    "product_meta_data_raw_header = product_meta_data_raw.take(1)[0]\n",
    "\n",
    "product_meta_data = product_meta_data_raw.filter(lambda line: line!=product_meta_data_raw_header).map(lambda line: line.split(\",\")).map(lambda tokens: (int(tokens[0]),tokens[1], str(tokens[2]))).cache()\n",
    "\n",
    "# print(product_meta_data_raw.take(3))\n",
    "print(product_meta_data.count())\n",
    "print(product_meta_data.take(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 554,
   "metadata": {},
   "outputs": [],
   "source": [
    "training_set, validation_set, test_set = ratings_data.randomSplit([6, 2, 2], seed=0L)\n",
    "validation_set_pred = validation_set.map(lambda x: (x[0], x[1]))\n",
    "test_set_pred = test_set.map(lambda x: (x[0], x[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TRAIN THE MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 495,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rank 8: RMSE = 1.63806409395\n",
      "Time required: 1791.45815802\n",
      "Best model rank: 8\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from pyspark.mllib.recommendation import ALS\n",
    "import math\n",
    "\n",
    "seed = 5L\n",
    "iterations = 20\n",
    "regularization_parameter = 0.5\n",
    "ranks = [8]\n",
    "errors = [0] * len(ranks)\n",
    "err = 0\n",
    "tolerance = 0.02\n",
    "\n",
    "min_error = float('inf')\n",
    "best_rank = -1\n",
    "best_iteration = -1\n",
    "for rank in ranks:\n",
    "    start = time.time()\n",
    "\n",
    "    model = ALS.train(training_set, rank, seed=seed, iterations=iterations,lambda_=regularization_parameter)\n",
    "    predictions = model.predictAll(validation_set_pred).map(lambda r: ((r[0], r[1]), r[2]))\n",
    "    ratings_predictions = validation_set.map(lambda r: ((int(r[0]), int(r[1])), float(r[2]))).join(predictions)\n",
    "    error = math.sqrt(ratings_predictions.map(lambda r: (r[1][0] - r[1][1])**2).mean())\n",
    "    errors[err] = error\n",
    "    err += 1\n",
    "    print('Rank %s: RMSE = %s' % (rank, error))\n",
    "    if error < min_error:\n",
    "        min_error = error\n",
    "        best_rank = rank\n",
    "        \n",
    "    end = time.time()\n",
    "    print('Time required: %s' % str(end - start))\n",
    "print('Best model rank: %s' % best_rank)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Optimize Regularization parameter\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 325,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "RP 0.001: RMSE = 7.44214225534\n",
      "Time required: 26.4670288563\n",
      "RP 0.005: RMSE = 5.7002634169\n",
      "Time required: 22.3995189667\n",
      "RP 0.01: RMSE = 5.20962179875\n",
      "Time required: 29.6096789837\n",
      "RP 0.05: RMSE = 4.28086486979\n",
      "Time required: 22.1840929985\n",
      "RP 0.1: RMSE = 3.70143706227\n",
      "Time required: 20.4284348488\n",
      "RP 0.5: RMSE = 3.06108986226\n",
      "Time required: 21.7208747864\n",
      "RP 1.0: RMSE = 3.12001839469\n",
      "Time required: 19.3582651615\n",
      "RP 5.0: RMSE = 4.47756841721\n",
      "Time required: 20.6618559361\n",
      "RP 10.0: RMSE = 4.47757492448\n",
      "Time required: 21.6949081421\n",
      "RP 50.0: RMSE = 4.47757492448\n",
      "Time required: 18.1499230862\n",
      "Best model rank: 0.5\n"
     ]
    }
   ],
   "source": [
    "import time\n",
    "from pyspark.mllib.recommendation import ALS\n",
    "import math\n",
    "\n",
    "seed = 5L\n",
    "iterations = 20\n",
    "regularization_parameters = [0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1.0, 5.0, 10.0, 50.0]\n",
    "rank = 8\n",
    "errors = [0] * len(regularization_parameters)\n",
    "err = 0\n",
    "tolerance = 0.02\n",
    "\n",
    "min_error = float('inf')\n",
    "best_reg_param = -1\n",
    "best_iteration = -1\n",
    "for regularization_parameter in regularization_parameters:\n",
    "    start = time.time()\n",
    "\n",
    "    model = ALS.train(training_set, rank, seed=seed, iterations=iterations,lambda_=regularization_parameter)\n",
    "    predictions = model.predictAll(validation_set_pred).map(lambda r: ((r[0], r[1]), r[2]))\n",
    "    ratings_predictions = validation_set.map(lambda r: ((int(r[0]), int(r[1])), float(r[2]))).join(predictions)\n",
    "    error = math.sqrt(ratings_predictions.map(lambda r: (r[1][0] - r[1][1])**2).mean())\n",
    "    errors[err] = error\n",
    "    err += 1\n",
    "    print('RP %s: RMSE = %s' % (regularization_parameter, error))\n",
    "    if error < min_error:\n",
    "        min_error = error\n",
    "        best_reg_param = regularization_parameter\n",
    "        \n",
    "    end = time.time()\n",
    "    print('Time required: %s' % str(end - start))\n",
    "print('Best model rank: %s' % best_reg_param)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### LOAD EXISTING MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 556,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/05/30 01:02:06 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Found 5 items\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-30 00:23 /models/model_Books_8\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-29 20:16 /models/model_Electronics_8\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-29 00:56 /models/model_Musical_Instruments_64\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-26 20:36 /models/model_Musical_Instruments_8\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-26 16:08 /models/model_Cell_Phones_and_Accessories_8\n"
     ]
    }
   ],
   "source": [
    "! hdfs dfs -ls -t /models"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 557,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.mllib.recommendation import MatrixFactorizationModel\n",
    "\n",
    "model_name = \"model_\"+str(category)+\"_\"+str(8)\n",
    "model_path = os.path.join(models_path, model_name)\n",
    "\n",
    "model = MatrixFactorizationModel.load(sc, model_path)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### TEST MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 558,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "For testing data the RMSE is 1.81602524824\n",
      "Time required: 27.3252410889\n"
     ]
    }
   ],
   "source": [
    "start = time.time()\n",
    "predictions = model.predictAll(test_set_pred).map(lambda r: ((r[0], r[1]), r[2]))\n",
    "ratings_predictions = test_set.map(lambda r: ((int(r[0]), int(r[1])), float(r[2]))).join(predictions)\n",
    "error = math.sqrt(ratings_predictions.map(lambda r: (r[1][0] - r[1][1])**2).mean())\n",
    "end = time.time()    \n",
    "print 'For testing data the RMSE is %s' % (error)\n",
    "print('Time required: %s' % str(end - start))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 559,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((170792, 91680), 3.9691310364186223),\n",
       " ((386896, 348712), 2.6288057284293873),\n",
       " ((1005400, 124660), 3.4531108164685413),\n",
       " ((988056, 78375), 4.056259395822714),\n",
       " ((196224, 239285), 3.6757123933378515)]"
      ]
     },
     "execution_count": 559,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predictions.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 560,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[((263259, 133999), (5.0, 2.4358965699487687)),\n",
       " ((896511, 83283), (1.0, 3.2342934307949154)),\n",
       " ((2719, 222819), (5.0, 4.099493528103354)),\n",
       " ((404739, 352343), (5.0, 0.6393902736572643)),\n",
       " ((729142, 27996), (5.0, -0.7395116837819558))]"
      ]
     },
     "execution_count": 560,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ratings_predictions.take(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 562,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[5, 1, 5, 5, 5, 5, 5, 4, 1, 5]\n",
      "650054\n",
      "(1, 54777)\n",
      "(2, 34817)\n",
      "(3, 54905)\n",
      "(4, 128220)\n",
      "(5, 377335)\n",
      "650054\n"
     ]
    }
   ],
   "source": [
    "actual_data_distribution = ratings_predictions.map(lambda x: int((float(x[1][0]))))\n",
    "print(actual_data_distribution.take(10))\n",
    "print(actual_data_distribution.count())\n",
    "\n",
    "actual_count_dict = dict()\n",
    "actual_count_sum = 0\n",
    "for i in range(1,6):\n",
    "    actual_data_distribution_group = actual_data_distribution.filter(lambda x: x==i)\n",
    "    count_i = actual_data_distribution_group.count()\n",
    "    actual_count_sum += count_i\n",
    "    actual_count_dict[i] = count_i\n",
    "    print(i, count_i)\n",
    "print(actual_count_sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 563,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEWCAYAAABliCz2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGkxJREFUeJzt3XmcXFWd9/HPNwQJkLBJYNgDT5AY\nIcAQUAZENnkAAzKvYQBX5MHJ4AqKoyLI6CMjjj4qOm5EQBg2WUbUhxEQEERcgG4IEEgAgSAhhCQQ\nQoIJS/KbP84pKJpeqrvrdnXX+b5fr3p11b237j2n6tb3njr39ilFBGZm1v5GtboAZmY2NBz4ZmaF\ncOCbmRXCgW9mVggHvplZIRz4ZmaFcOAPkqQfSfpik9a1taTlktbIj2+W9OFmrDuv7xpJxzZrff3Y\n7hmSFktaMETba1U9z5R00lBvty+Szpd0Rr4/RdIfKt5eSJpY5TYGq+tnrRQO/F5ImitphaRlkp6V\n9AdJJ0h65XWLiBMi4isNruvA3paJiL9ExNiIWNWEsn9J0kVd1n9IRFww2HX3sxxbAycDkyPib7qZ\nv6+k1fnDt0zSA5KO68f6h0s9xwMfBM7uMn3bXL8f9mNdr6tTs0TEPcCzkg4b6Dpy+V7K71nt9mwT\ni9n0xk5XzfysjSQO/L4dFhHjgG2ArwGfA85t9kYkjW72OoeJrYGnI2JhL8vMj4ixwHrAp4AfS9ph\nSErXPB8CfhURK7pM/yCwBDha0lpDXqruXQz88yDXcVkOzNptg2YUrFFt/HmpVkT41sMNmAsc2GXa\nHsBqYMf8+HzgjHx/Y+Bq4FngGeB3pIPqhfk5K4DlwGeBCUAAxwN/AW6pmzY6r+9m4EzgduA54BfA\nRnnevsC87soLHAy8CLyUt3d33fo+nO+PAk4DHgMWAv8JrJ/n1cpxbC7bYuDUXl6n9fPzF+X1nZbX\nf2Cu8+pcjvO7eW539VgI/GPd4+8Aj+fXoBN4e57eSD0/BNwK/D9S8D4KHFK37m3za78MuAH4PnBR\nnjcGuAh4Or+ndwCb9vAa/AZ4f5dpAh4GPgI8BRzZZf5bgOvzvvIU8IVe6jSXun0R+FKtnPnxFcAC\nYGmuz1vq5p1P3kfz4y3y+7LWAD8Xr9l2N/MDmJjvr5Vf+7/kOv4IWLtu2XcDM/N7+3Cu/78Bq4CV\n+TX4Xt16PwY8BDyap/1dfl+W5r9/V7fum4GvAL/P7++vgY277OO1z9pGwE+A+Xk/+Xlvn+lWZ9NA\nb27h91NE3A7MA97ezeyT87zxwKakD3BExAdIO/xhkVpDX697zjuANwP/u4dNfhD4P8BmwMvAdxso\n47XAV3m1FbZzN4t9KN/2A7YDxgLf67LM3sAOwAHA6ZLe3MMm/4MU+tvl+nwQOC4ibgAOIbfgI+JD\nvZVb0ihJh5M+ZH+um3UHsAvpQ3kJcIWkMQ3WE+CtwAN5vV8HzpWkPO8S0gH1jaQg+0Dd847N9doq\nzz+BFJTd2Slvo97ewJbAT4HL8/pqdR1HOsBcC2wOTARu7EeduroG2B7YBLiT1IrvVkQ8QTqgDMW3\nqK8BbyK9fxNJB5vTASTtQWoo/AuwAbAPMDciTiUF68fza/DxuvUdQXo/J0vaCPhv0mfijcC3gP+W\n9Ma65d8LHEd6Xd4AfKaHcl4IrEM6CG8CfDtP7/YzPZAXYjhw4A/MfFL4dPUSKZi3iYiXIuJ3kZsJ\nvfhSRDwfr+8KqLkwImZFxPPAF4GjmnSi6X3AtyLikYhYDpwCHNPlq/KXI2JFRNwN3A28LnxyWY4B\nTomIZRExF/gmrw3Ovmye+4BXAFcBn46Iu2ozI+KiiHg6Il6OiG+SWo39CavHIuLHkfprLyC9R5vm\n8wu7A6dHxIsRcSvwy7rnvUQKkokRsSoiOiPiuR62sQGpFVnvWOCaiFhCOrAcLGmTPG8asCAivhkR\nK/Nrd1s/6vQaEXFeXscLpAPXzpLW7+Upy3KZB+qofF6rdrup6wL5oDod+FREPBMRy0gHs2PyIscD\n50XE9RGxOiKeiIg5fWz3zLyuFcC7gIci4sK8b1wKzAHqz0/8JCIezMtfTjrwdC3nZqSGyQkRsSR/\ndn+bZw/kMz1sOfAHZgvS17uuvkFqmf5a0iOSPt/Auh7vx/zHgDVJLdXB2jyvr37do0mtmJr6q2r+\nSvoW0NXGuUxd17VFP8oyP1If8Hqk1tr+9TMlfUbSbElL84Fhffr3GrxSj4j4a747lvQaPFM3DV77\nel8IXAf8VNJ8SV+XtGYP21gCjKsr89rAP5Jb2hHxR9K3vPfmRbYidWEMmqQ1JH1N0sOSniN1/0Dv\nr9E4UjdF13W9ve5E7H29PP/yiNig7rZfN8uMJ7WaO2sHBtI3mvF5/kBeg/r3p+s+DK/f9xrZh7ci\n7QdLupk3kM/0sOXA7ydJu5N2qFu7zsstrJMjYjvgcODTkg6oze5hlX21Fraqu781qcWxGHie9GGq\nlWsNXv0gNbLe+aQT0fXrfpnUz9ofi3OZuq7riX6uh9w6/Rywk6QjIAUQ6ZzHUcCG+cCwlNQ/DoP7\nev0ksJGkdeqmvfJ65xbdlyNiMqmveBqpu6o795C6Lmr+nnQA+4GkBfmS1C14tVvncVIXWHe6q9Nr\n3m+g/oqn95L6wg8kHQwn5OmiG5K2IHVvdO2CIrdgaydi39JD+Rq1mPSt7S11B4b1I52gh/Qa/K8e\nntvI56XrPgwD2/ceJ+0Hr/vG08dnesRx4DdI0nqSppH6Yy+KiHu7WWaapIn5q+xS0omn1Xn2U/T8\nAe/N+yVNzqH0f4Erc9fEg8AYSe/Krc7TSF0dNU8BE+ovIe3iUuBT+bLBsbzab/xyfwqXy3I58G+S\nxknaBvg06WRnv0XEi6QuodPzpHGkA9EiYLSk00lBWtNXPXvb1mNAB/AlSW+QtCd13QGS9pO0Uz6Y\nPkc6sK3ufm38inT+ouZY4DxS3/4u+bYXqatlJ9KJwM0knSRprfzavbWXOs0kdbmtKWkqcGTdvHHA\nC6STy+uQ3svevAP4TT7AViYiVgM/Br5d68qStIWk2vmqc4HjJB2Qz99sIWlSntfI5+VXwJskvVfS\naElHA5NJr21/yvkk6RzIDyRtmF/jfXJ5e/tMjzgO/L79f0nLSK2AU0knhnq6Tnx70om45cAfgR9E\nRK1v80zgtPzVtqcTR925kHSVxQLSVSOfBIiIpcBHgXNILZrnSSeXaq7If5+WdGc36z0vr/sW0pUr\nK4FP9KNc9T6Rt/8I6ZvPJXn9A3UesHW+Vvw6UjfAg6Sv6yt57df6vurZl/cBe5LC8gzgMlJ4QmpF\nX0kK+9nAb0mvWXf+EzhU0tq5BX0AcFZELKi7dea6HJv7s99JOsAsIF15UusW6a5OXyS1hpcAXya9\nxvXbfoy0H9wP/KmBOv+oj2X6cnSX6/CX152fqPc5UpfIn3J30w3k8y/5AojjSCdIl5Je31qL/TvA\nkZKWSOr2QoWIeJr0retk0vv3WWBaRCweQH0+QDqgzyFdJVb7B7rePtMjjkbw+QezppN0GTAnIv51\nAM/9KrAwIs5qfsmaQ9IU4OyI2LPVZbGh58C3ouVzMs+QvuUcBPwc2LP+KiGzduH/VrPS/Q3wM9Ll\nl/OAjzjsrV25hW9mVgiftDUzK8Sw6tLZeOONY8KECa0uhpnZiNHZ2bk4Isb3veQwC/wJEybQ0dHR\n6mKYmY0Ykrr+t3GP3KVjZlYIB76ZWSEc+GZmhXDgm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kVYlj9\n4xUrO2FOtz/SY2bWniYN3XhmbuGbmRXCgW9mVggHvplZIRz4ZmaFcOCbmRXCgW9mVggHvplZIRz4\nZmaFcOCbmRXCgW9mVggHvplZIRz4ZmaFcOCbmRWi0tEyJc0FlgGrgJcjYmqV2zMzs54NxfDI+0XE\n4iHYjpmZ9cJdOmZmhag68AP4taROSdO7W0DSdEkdkjoWLam4NGZmBau6S2fviHhC0ibA9ZLmRMQt\n9QtExAxgBsDUHTV0P/1iZlaYSlv4EfFE/rsQuArYo8rtmZlZzyoLfEnrShpXuw8cBMyqantmZta7\nKrt0NgWuklTbziURcW2F2zMzs15UFvgR8Qiwc1XrNzOz/vFlmWZmhXDgm5kVwoFvZlYIB76ZWSEc\n+GZmhXDgm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kVYih+8apxY3aDSR2tLoWZWVtyC9/MrBAOfDOz\nQjjwzcwK4cA3MyuEA9/MrBAOfDOzQjjwzcwKMbyuw1/ZCXPU6lJYTyZFq0tgZoPgFr6ZWSEc+GZm\nhXDgm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kVwoFvZlYIB76Z\nWSEqD3xJa0i6S9LVVW/LzMx6NhQt/BOB2UOwHTMz60WlgS9pS+BdwDlVbsfMzPpWdQv/LOCzwOqe\nFpA0XVKHpI5FSyoujZlZwSoLfEnTgIUR0dnbchExIyKmRsTU8RtWVRozM6uyhb8XcLikucBPgf0l\nXVTh9szMrBeVBX5EnBIRW0bEBOAY4DcR8f6qtmdmZr3zdfhmZoUYPRQbiYibgZuHYltmZtY9t/DN\nzArhwDczK4QD38ysEA58M7NCOPDNzArhwDczK4QD38ysEA58M7NCOPDNzArhwDczK8SQDK3QsDG7\nwaSOVpfCzKwtuYVvZlYIB76ZWSEc+GZmhXDgm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kVYnj949XK\nTpijVpeicZOi1SUwM2uYW/hmZoVw4JuZFaKhLh1J3+1m8lKgIyJ+0dwimZlZFRpt4Y8BdgEeyrcp\nwJbA8ZLOqqhsZmbWRI2etJ0C7BURqwAk/RD4HbA3cG9FZTMzsyZqtIW/ITC27vG6wEb5APBC00tl\nZmZN12gL/+vATEk3AwL2Ab4qaV3ghorKZmZmTdRQ4EfEuZJ+BeyRJ30hIubn+/9SScnMzKyp+nNZ\n5ihgEbAEmChpn2qKZGZmVWj0ssx/B44G7gNW58kB3FJRuczMrMka7cM/AtghInyC1sxshGq0S+cR\nYM0qC2JmZtVqtIX/V9JVOjdSdxlmRHyypydIGkPq8lkrb+fKiPjXQZTVzMwGodHA/2W+9ccLwP4R\nsVzSmsCtkq6JiD/1cz1mZtYEjV6WeUF/VxwRASzPD9fMN48nbGbWIr0GvqTLI+IoSffSTVhHxJQ+\nnr8G0AlMBL4fEbd1s8x0YDrA1pv3o+RmZtYvfbXwT8x/pw1k5XnohV0kbQBcJWnHiJjVZZkZwAyA\nqTvK3wDMzCrS61U6EfFkvvvRiHis/gZ8tNGNRMSzwE3AwQMvqpmZDUajl2W+s5tph/T2BEnjc8se\nSWvndczpX/HMzKxZ+urD/wipJb+dpHvqZo0Dft/HujcDLsj9+KOAyyPi6sEU1szMBq6vPvxLgGuA\nM4HP101fFhHP9PbEiLgH2HVwxTMzs2bpNfAjYinppwzfAyBpE9KvX42VNDYi/lJ9Ec3MrBka6sOX\ndJikh4BHgd8Cc0ktfzMzGyEaPWl7BvA24MGI2BY4APB/zJqZjSCNBv5LEfE0MErSqIi4CZhaYbnM\nzKzJGh1L51lJY0mDoV0saSHwfHXFMjOzZmu0hf9u0oiZnwKuBR4GDquqUGZm1nyNDp5Wa82vJl1b\nP4p05c7FVRXMzMyaq9cWvqT1JJ0i6XuSDlLycdIPohw1NEU0M7Nm6KuFfyHpR8v/CHwY+AIg4IiI\nmFlx2czMrIn6CvztImInAEnnAE8CW0fEyspLZmZmTdXXSduXanfyUMfzHPZmZiNTXy38nSU9l+8L\nWDs/FulHrdZramnG7AaTOpq6SjMzS/oaS2eNoSqImZlVq9Hr8M3MbIRz4JuZFcKBb2ZWCAe+mVkh\nHPhmZoVw4JuZFaLR4ZGHxspOmKOh2dakGJrtmJkNE27hm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kV\nwoFvZlYIB76ZWSEc+GZmhXDgm5kVwoFvZlYIB76ZWSEc+GZmhags8CVtJekmSfdLuk/SiVVty8zM\n+lblaJkvAydHxJ2SxgGdkq6PiPsr3KaZmfWgshZ+RDwZEXfm+8uA2cAWVW3PzMx6NyR9+JImALsC\nt3Uzb7qkDkkdi5YMRWnMzMpUeeBLGgv8F3BSRDzXdX5EzIiIqRExdfyGVZfGzKxclQa+pDVJYX9x\nRPysym2ZmVnvqrxKR8C5wOyI+FZV2zEzs8ZU2cLfC/gAsL+kmfl2aIXbMzOzXlR2WWZE3AoM0S+S\nm5lZX/yftmZmhXDgm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kVwoFvZlYIB76ZWSEc+GZmhXDgm5kV\nwoFvZlaIKn/isP/G7AaTOlpdCjOztuQWvplZIRz4ZmaFcOCbmRXCgW9mVggHvplZIRz4ZmaFcOCb\nmRVieF2Hv7IT5oygn8GdFK0ugZlZw9zCNzMrhAPfzKwQDnwzs0I48M3MCuHANzMrhAPfzKwQDnwz\ns0I48M3MCuHANzMrhAPfzKwQDnwzs0I48M3MClFZ4Es6T9JCSbOq2oaZmTWuyhb++cDBFa7fzMz6\nobLAj4hbgGeqWr+ZmfVPy/vwJU2X1CGpY9GSVpfGzKx9tTzwI2JGREyNiKnjN2x1aczM2lfLA9/M\nzIaGA9/MrBBVXpZ5KfBHYAdJ8yQdX9W2zMysb5X9iHlEvKeqdZuZWf+5S8fMrBAOfDOzQjjwzcwK\n4cA3MyuEA9/MrBAOfDOzQjjwzcwK4cA3MyuEA9/MrBAOfDOzQjjwzcwKUdlYOgMyZjeY1NHqUpiZ\ntSW38M3MCuHANzMrhAPfzKwQDnwzs0I48M3MCuHANzMrhAPfzKwQDnwzs0I48M3MCqGIaHUZXiFp\nGfBAq8tRkY2Bxa0uREVct5GnXesF5dVtm4gY38iTh9fQCvBARExtdSGqIKnDdRt52rVu7VovcN16\n4y4dM7NCOPDNzAox3AJ/RqsLUCHXbWRq17q1a73AdevRsDppa2Zm1RluLXwzM6uIA9/MrBDDIvAl\nHSzpAUl/lvT5VpdnMCSdJ2mhpFl10zaSdL2kh/LfDVtZxoGStJWkmyTdL+k+SSfm6SO+fpLGSLpd\n0t25bl/O07eVdFveNy+T9IZWl3WgJK0h6S5JV+fHbVE3SXMl3StppqSOPG3E75MAkjaQdKWkOZJm\nS9pzMHVreeBLWgP4PnAIMBl4j6TJrS3VoJwPHNxl2ueBGyNie+DG/Hgkehk4OSImA28DPpbfq3ao\n3wvA/hGxM7ALcLCktwH/Dnw7IiYCS4DjW1jGwToRmF33uJ3qtl9E7FJ3jXo77JMA3wGujYhJwM6k\n92/gdYuIlt6APYHr6h6fApzS6nINsk4TgFl1jx8ANsv3NyP9g1nLy9mEev4CeGe71Q9YB7gTeCvp\nvxpH5+mv2VdH0g3YMofD/sDVgNqobnOBjbtMG/H7JLA+8Cj54ppm1K3lLXxgC+Dxusfz8rR2smlE\nPJnvLwA2bWVhmkHSBGBX4DbapH65y2MmsBC4HngYeDYiXs6LjOR98yzgs8Dq/PiNtE/dAvi1pE5J\n0/O0dtgntwUWAT/JXXHnSFqXQdRtOAR+USIdlkf0tbCSxgL/BZwUEc/VzxvJ9YuIVRGxC6k1vAcw\nqcVFagpJ04CFEdHZ6rJUZO+I+FtSt/DHJO1TP3ME75Ojgb8FfhgRuwLP06X7pr91Gw6B/wSwVd3j\nLfO0dvKUpM0A8t+FLS7PgElakxT2F0fEz/LktqkfQEQ8C9xE6ubYQFJtzKmRum/uBRwuaS7wU1K3\nzndoj7oREU/kvwuBq0gH63bYJ+cB8yLitvz4StIBYMB1Gw6Bfwewfb5i4A3AMcAvW1ymZvslcGy+\nfyyp73vEkSTgXGB2RHyrbtaIr5+k8ZI2yPfXJp2bmE0K/iPzYiOybhFxSkRsGRETSJ+v30TE+2iD\nuklaV9K42n3gIGAWbbBPRsQC4HFJO+RJBwD3M5i6tfrERD7xcCjwIKnP9NRWl2eQdbkUeBJ4iXSE\nPp7UX3oj8BBwA7BRq8s5wLrtTfr6eA8wM98ObYf6AVOAu3LdZgGn5+nbAbcDfwauANZqdVkHWc99\ngavbpW65Dnfn2321/GiHfTLXYxegI++XPwc2HEzdPLSCmVkhhkOXjpmZDQEHvplZIRz4ZmaFcOCb\nmRXCgW9mVggHvo1YklblERJnSbpC0jotKsdJrdq2WX848G0kWxFphMQdgReBExp9Yh6ltVlOIg24\nZjasOfCtXfwOmAgg6f15bPuZks6uhbuk5ZK+KeluYE9Ju0v6Qx4D/3ZJ4/IAat+QdIekeyT9c37u\nvpJurhub/GIlnwQ2B26SdFNe9oeSOurH1c/TD83P7ZT03bpx6ddV+h2F2/MgWe8e2pfOSuHAtxEv\njwdzCHCvpDcDRwN7RRoIbRXwvrzousBtkca8vx24DDgxPz4QWEH6z+ilEbE7sDvwT5K2zc/fldSa\nn0z6D8+9IuK7wHzSeOz75eVOjTQu+xTgHZKmSBoDnA0cEhG7AePrqnAqabiDPYD9gG/kYQLMmmp0\n34uYDVtr5+GMIbXwzwWmA7sBd6Shf1ibVweXWkUa+A1gB+DJiLgDIPKon5IOAqZIqo0xsz6wPanL\n6PaImJeXm0n63YNbuynXUXmY3tGk8conkxpXj0TEo3mZS3NZIY3/crikz+THY4Ctee2PlZgNmgPf\nRrIVuRX/ijzA2wURcUo3y6+MiFV9rFPAJyLiui7r3Zf0q1g1q+jm85O/DXwG2D0ilkg6nxTgfW3z\nHyLigT6WMxsUd+lYu7kROFLSJvDKb5tu081yDwCbSdo9Lzcudw1dB3wkDwONpDc10L2yDBiX769H\nGrd8qaRNSV1Nte1tl384BlK3U811wCfywQpJuzZaWbP+cAvf2kpE3C/pNNIvII0ijVr6MeCxLsu9\nKOlo4D/ycMgrSP3455C6au7MAbwIOKKPzc4ArpU0PyL2k3QXMIf0S26/z9tbIemjebnnScOC13yF\n9ItU9+QyPwpMG/CLYNYDj5ZpNkQkjY2I5flA8n3goYj4dqvLZeVwl47Z0PmnfLL3PtLJ4LNbXB4r\njFv4ZmaFcAvfzKwQDnwzs0I48M3MCuHANzMrhAPfzKwQ/wPuNg5vWIzUkwAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x115735f50>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "values = actual_count_dict.values()\n",
    "plot_values = (x*100/actual_count_sum for x in values)\n",
    "plt.barh(list(actual_count_dict.keys()), list(plot_values), color='gold')\n",
    "plt.ylabel(\"Rating\")\n",
    "plt.xlabel(\"Percentage\")\n",
    "plt.title(\"Distribution of Ratings (Actual) - \"+str(category))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 564,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[3, 4, 5, 1, 0, 5, 4, 3, 1, 3]\n",
      "650054\n",
      "(1, 39340)\n",
      "(2, 44042)\n",
      "(3, 110128)\n",
      "(4, 272227)\n",
      "(5, 168052)\n",
      "633789\n"
     ]
    }
   ],
   "source": [
    "prediction_data_distribution = ratings_predictions.map(lambda x: int(math.ceil(float(x[1][1]))))\n",
    "print(prediction_data_distribution.take(10))\n",
    "print(prediction_data_distribution.count())\n",
    "\n",
    "prediction_count_dict = dict()\n",
    "prediction_count_sum = 0\n",
    "for i in range(1,6):\n",
    "    prediction_data_distribution_group = prediction_data_distribution.filter(lambda x: x==i)\n",
    "    count_i = prediction_data_distribution_group.count()\n",
    "    prediction_count_sum += count_i\n",
    "    prediction_count_dict[i] = count_i\n",
    "    print(i, count_i)\n",
    "print(prediction_count_sum)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 565,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXwAAAEWCAYAAABliCz2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAGztJREFUeJzt3Xu4HFWZ7/HvjxAIJOESCQgkISAO\nkUEuElAOiIDKAQR0nqPAjHhgDpgBZxQGkAFBxXPwfrwxOiqiE+4XUUaGwQsKEVAPsAMBAgRBLgYS\nSAJJSJBwSd7zx1obOpu9e3cnXbs6Wb/P8/Szu+uy6q3VVW+tWlW7WhGBmZmt/dapOwAzMxsaTvhm\nZoVwwjczK4QTvplZIZzwzcwK4YRvZlYIJ/xM0vckfbpDZU2QtFTSsPx5mqTjO1F2Lu/nko7pVHlt\nLPdcSQskPTVEy6trPb8o6eQhWtY5ki7J71fabtos51OSLuh8hCst43ZJf11h+VMlnVtV+Z1S13bZ\nERGx1r+Ax4AXgCXAIuD3wAnAOqtY1nvanGcacPwqxn4OcEkX1OGEXIebDzB+P2AFsDTX84PA36+B\n6zkWeBLYoBPrVcV655ieqKFujgB+shrzTwQi12Xj68g8fipw7mrGeCxwa93bUbe+1l2lo8Sa6bCI\n+LWkjYF3Ad8C3g78fScXImndiHilk2V2iQnAMxExr8k0cyJinCQBBwPXSvp9RDw4NCF2xLHA9RHx\nQsOwxvV6P3C1pNsi4v7GGdfi777XtcD3JL0xIlbnLG+TOutJ0rCIWF7X8mtV9xFnKF700yoH9iS1\n3HbKn6eSWxfAZsB1pLOBZ4FbSN1fF+d5XiC1TE7ntVbLccCfgZsbhq2by5sGfBG4HXgO+BkwJo/b\njz6ttd54gYOAl4CX8/Lubijv+Px+HeBs4HFgHnARsHEe1xvHMTm2BcBZTepp4zz//Fze2bn89+R1\n7m3pTu1n3v7WYx7woYbP3wJm5zqYDrwzD29lPY8FbgX+L7AQeBQ4uKHsbXPdLwF+DXyH3HIGRgCX\nAM/k7/QOYIsB6uBG4OhB1ms+8MH+vvs8/h2ks8hFwN3Afn3i/G2O8wbg2w1x9pbXu92MAf4dmJPX\n+T+AkX2+i6XAVvQ5UwAOB+7LMUwD3tJn+zoNuAdYDFwJjGi27TfMewNwzCruhyutXz/jp9LQwgcO\nBWbw2ln5zg3jxgM/zd/FM7ke3wIsA5bnelnUUO53geuB50nbc7/beovb2jQaztiBjwIP5O/0fuBt\nefi/kM4We88M3117Lqw7gCFZyQG6YfJOemLfjY2UnL8HDM+vdwLqr6yGjfiivDNu0HfDzhvIk8BO\neZqf8NpOvh8DJPz8/hz6nPKzciL8X8DDwHbAqLwTXNwnth/kuHYBXqRh5+9T7kWkg9HoPO8fgeMG\nirPPvK+OJx0kDiclpd0apjkaeAOwLnAq8BSvJZrB1vNY0gHho8Aw4ERSIuz9Xv5A2kHXA/YhHVR6\n6/gfgP8ENszz7g5sNMB6zAf2aLJef5Pj2GGA735rUgI6JE//3vx5bEOcXwfWB/YlJYOBEv5/kZLx\npqTt8F1NtplzGsr5K1Jie2+e73TSNrJew/Z1O+lAMYaUrE4YbNvP488Dvr6K++FK69fP+Km8tg/u\nRmowvD1/Z8fkuNfPn+8GvpHrfQSwT8N2cms/5S4G9s7fyQiab+vH0nxbm8Zr2+WHSPv2HoCA7YFt\n8vYxG9iqYd3fVHcuLP2i7RzSBt/Xy8CWwDYR8XJE3BL5W2vinIh4PlbuCmh0cUTMjIjngU8DR6zK\nxbl+fJi0Az4SEUuBM4GjJDV2130uIl6IiLtJO8oufQvJsRwFnBkRSyLiMeBrwEfaiGUrSYtILdBr\ngFMi4q7ekRFxSUQ8ExGvRMTXSDvvDm2U/3hE/CDS6fiFpO9oC0kTSDvcZyLipYi4ldT90Otl0oFm\n+4hYHhHTI+K5AZaxCSkJ97deC4DPAh+JlbupGr/7o0ldQtdHxIqIuAHoAQ5piPPTEfFiRNxMOhC9\njqQtSd1iJ0TEwrwd/rbFejoS+K+IuCEiXiYdCDcA/lvDNOdFxJyIeDbHsGsePti2vyTX0epYIGlR\nw+st/UwzBfh+RNyWv7MLSY2Vd5DOzrcCPpnrfVn+zpv5WUT8LiJW5HUcbFvvd1vrp9zjga9ExB2R\nPBwRj5POMtYHdpQ0PCIei4g/tVg/lSk94W9NOm3t66ukFtGvJD0i6YwWyprdxvjHSa2nzVqKsrmt\ncnmNZa/LyhtnY3/rX0hnAn1tlmPqW9bWbcQyJyI2ATYitQQPaBwp6TRJD0hanBPoxrRXB6+uR0T8\nJb8dRaqDZxuGwcr1fTHwS+AKSXMkfUXS8AGWsZDU6nvdekXEmIjYNSKu6DO+cVnbAB9qTGikM44t\nc5wL80G/V2N9Nxqf12nhAOObWWmbyEluNit/lwNtE4Nt+6NJXSyvI+m+fJfRUknvbBLfZrk+e18P\n9DPNNsCpfepxfF638aSE3M51gMbvqJVtfaBtra/xwOsSeUQ8DJxMOvOaJ+kKSVu1EW8lik34kvYg\nfcGvaxnko/6pEbEdqWviFEnv7h09QJGDnQGMb3g/gdTKWEA69d6wIa5hpDtFWi13DmnnaCz7FeDp\nQebra0GOqW9ZT7ZZDhHxIqn/8q2SPgCQE8DppDs9Ns0HhsWk02AYfD2bmQuMkbRhw7BX6zu3VD8X\nETuSWrmHAv9zgLLuIXWJtKMx9tmks7nGhDYyIr6U49xU0siG6ScMUObsvE79tabb2ibyxebxtPBd\nDrLtQ+onv3uAef86Ikbl1y2DLWsQs4HP96nHDSPi8jxuQp+z2FfDGKC8xuEd29ZzLG/qd4ERl0XE\nPnk5AXx5FcrvqOISvqSNJB0KXEHq87y3n2kOlbR93lEWk07PVuTRT5P6y9t1tKQdc1L638DV+XTx\nj8AISe/Lrc6zSaeCvZ4GJkoa6Lu6HPhnSdtKGgV8AbiyzdYPOZargM9LGi1pG+AU0sXOtkXES6TT\n5M/kQaNJB6L5wLqSPkM6E+g12Ho2W9bjpG6TcyStJ2kv4LDe8ZL2l/TWfDB9jrSzr+i/NK4n3cW1\nqi4BDpP03yUNkzRC0n6SxjXE+bkc5z6NcfZZp7nAz4F/k7SppOGS9s2jnwbekO84689VwPskvTtv\nU6eSukN+P1jwzbZ9SSNI1z9uaK0qVssPgBMkvV3JyLyPjCZdf5gLfCkPHyFp7zzf08A4SesNVHCH\nt/ULgNMk7Z7j3F7SNpJ2kHSApPVJF5J7L7TXqqSE/5+SlpCOyGeRLpwNdEvmm0l3eiwlXWT7t4i4\nKY/7InB2Ps08rY3lX0y6ePQU6aLRJwAiYjHwMdKG8ySpxf9Ew3w/zn+fkXRnP+X+KJd9M+lugmXA\nx9uIq9HH8/IfIZ35XJbLX1U/IrXEDiN1qfyCdIB7PMfZeJo92HoO5sPAXqQLpOeSLna+mMe9Ebia\nlOwfIN0lc/EA5VxE6m/fYBViICJmk27d/BTp4DYb+CSv7Wt/R7oQ+SzpesBFTYr7COngNIt0AfPk\nvIxZpAP9I3k7XKmrIF9fOBr4V1Jr9jDSbckvtbAKzbb9w4BpETGnhXKaWdTQ9bNU0il9J4iIHtJF\n02+TutkeJl1M7U3Yh5EukP6ZtL8cmWe9kXR30lOSFjSJoSPbekT8GPh8nn8J6U6qMaRG25dI9f8U\nsDnp+lqteq86m61VJF0JzIqIz67CvF8A5kXENzsf2ZpL0m2kO1lm1h2LrRonfFsr5Gsyz5LOcg4k\ntbT2arxLyKx0Jf2nra3d3kj6H4Q3kE7xT3SyN1uZW/hmZoUo6aKtmVnRuqpLZ7PNNouJEyfWHYaZ\n2Rpj+vTpCyJi7OBTdlnCnzhxIj09PXWHYWa2xpA00H9rv467dMzMCuGEb2ZWCCd8M7NCOOGbmRXC\nCd/MrBBO+GZmhXDCNzMrhBO+mVkhuuofr1g2HWZp8OnMSjHJz7qyznEL38ysEE74ZmaFcMI3MyuE\nE76ZWSGc8M3MCuGEb2ZWCCd8M7NCOOGbmRXCCd/MrBBO+GZmhXDCNzMrhBO+mVkhnPDNzApR6dMy\nJT0GLAGWA69ExOQql2dmZgMbiscj7x8RC4ZgOWZm1oS7dMzMClF1wg/gV5KmS5rS3wSSpkjqkdQz\nf2HF0ZiZFazqLp19IuJJSZsDN0iaFRE3N04QEecD5wNM3kn+eR8zs4pU2sKPiCfz33nANcCeVS7P\nzMwGVlnClzRS0uje98CBwMyqlmdmZs1V2aWzBXCNpN7lXBYRv6hweWZm1kRlCT8iHgF2qap8MzNr\nj2/LNDMrhBO+mVkhnPDNzArhhG9mVggnfDOzQjjhm5kVwgnfzKwQTvhmZoVwwjczK4QTvplZIYbi\nF69aN2J3mNRTdxRmZmslt/DNzArhhG9mVggnfDOzQjjhm5kVwgnfzKwQTvhmZoVwwjczK0R33Ye/\nbDrMUt1RmJkNnUkxZItyC9/MrBBO+GZmhXDCNzMrhBO+mVkhnPDNzArhhG9mVggnfDOzQjjhm5kV\nwgnfzKwQTvhmZoVwwjczK4QTvplZIZzwzcwKUXnClzRM0l2Srqt6WWZmNrChaOGfBDwwBMsxM7Mm\nKk34ksYB7wMuqHI5ZmY2uKpb+N8ETgdWDDSBpCmSeiT1zF9YcTRmZgWrLOFLOhSYFxHTm00XEedH\nxOSImDx206qiMTOzKlv4ewOHS3oMuAI4QNIlFS7PzMyaqCzhR8SZETEuIiYCRwE3RsTRVS3PzMya\n8334ZmaFWHcoFhIR04BpQ7EsMzPrn1v4ZmaFcMI3MyuEE76ZWSGc8M3MCuGEb2ZWCCd8M7NCOOGb\nmRXCCd/MrBBO+GZmhXDCNzMrxJA8WqFlI3aHST11R2FmtlZyC9/MrBBO+GZmhXDCNzMrhBO+mVkh\nnPDNzArhhG9mVggnfDOzQjjhm5kVorv+8WrZdJiluqOwOk2KuiMwW2u5hW9mVggnfDOzQrTUpSPp\nvH4GLwZ6IuJnnQ3JzMyq0GoLfwSwK/BQfu0MjAOOk/TNimIzM7MOavWi7c7A3hGxHEDSd4FbgH2A\neyuKzczMOqjVFv6mwKiGzyOBMfkA8GLHozIzs45rtYX/FWCGpGmAgH2BL0gaCfy6otjMzKyDWkr4\nEfFDSdcDe+ZBn4qIOfn9JyuJzMzMOqqd2zLXAeYDC4HtJe1bTUhmZlaFVm/L/DJwJHAfsCIPDuDm\niuIyM7MOa7UP/wPADhHhC7RmZmuoVrt0HgGGVxmImZlVq9UW/l9Id+n8hobbMCPiEwPNIGkEqctn\n/bycqyPis6sRq5mZrYZWE/61+dWOF4EDImKppOHArZJ+HhH/r81yzMysA1q9LfPCdguOiACW5o/D\n88vPvjUzq0nThC/pqog4QtK99JOsI2LnQeYfBkwHtge+ExG39TPNFGAKwISt2ojczMzaMlgL/6T8\n99BVKTw/emFXSZsA10jaKSJm9pnmfOB8gMk7yWcAZmYVaXqXTkTMzW8/FhGPN76Aj7W6kIhYBNwE\nHLTqoZqZ2epo9bbM9/Yz7OBmM0gam1v2SNoglzGrvfDMzKxTBuvDP5HUkt9O0j0No0YDvxuk7C2B\nC3M//jrAVRFx3eoEa2Zmq26wPvzLgJ8DXwTOaBi+JCKebTZjRNwD7LZ64ZmZWac0TfgRsZj0U4Z/\nCyBpc9KvX42SNCoi/lx9iGZm1gkt9eFLOkzSQ8CjwG+Bx0gtfzMzW0O0etH2XOAdwB8jYlvg3YD/\nY9bMbA3SasJ/OSKeAdaRtE5E3ARMrjAuMzPrsFafpbNI0ijSw9AulTQPeL66sMzMrNNabeG/n/TE\nzH8GfgH8CTisqqDMzKzzWn14Wm9rfgXp3vp1SHfuXFpVYGZm1llNW/iSNpJ0pqRvSzpQyT+RfhDl\niKEJ0czMOmGwFv7FpB8t/wNwPPApQMAHImJGxbGZmVkHDZbwt4uItwJIugCYC0yIiGWVR2ZmZh01\n2EXbl3vf5EcdP+Fkb2a2Zhqshb+LpOfyewEb5M8i/ajVRh2NZsTuMKmno0WamVky2LN0hg1VIGZm\nVq1W78M3M7M1nBO+mVkhnPDNzArhhG9mVggnfDOzQjjhm5kVotXHIw+NZdNhluqOoj2Tou4IzMxa\n4ha+mVkhnPDNzArhhG9mVggnfDOzQjjhm5kVwgnfzKwQTvhmZoVwwjczK4QTvplZIZzwzcwK4YRv\nZlYIJ3wzs0JUlvAljZd0k6T7Jd0n6aSqlmVmZoOr8mmZrwCnRsSdkkYD0yXdEBH3V7hMMzMbQGUt\n/IiYGxF35vdLgAeAratanpmZNTckffiSJgK7Abf1M26KpB5JPfMXDkU0ZmZlqjzhSxoF/AQ4OSKe\n6zs+Is6PiMkRMXnsplVHY2ZWrkoTvqThpGR/aUT8tMplmZlZc1XepSPgh8ADEfH1qpZjZmatqbKF\nvzfwEeAASTPy65AKl2dmZk1UdltmRNwKrGG/SG5mtvbyf9qamRXCCd/MrBBO+GZmhXDCNzMrhBO+\nmVkhnPDNzArhhG9mVggnfDOzQjjhm5kVwgnfzKwQTvhmZoWo8icO2zdid5jUU3cUZmZrJbfwzcwK\n4YRvZlYIJ3wzs0I44ZuZFcIJ38ysEE74ZmaFcMI3MytEd92Hv2w6zFrDfgZ3UtQdgZlZS9zCNzMr\nhBO+mVkhnPDNzArhhG9mVggnfDOzQjjhm5kVwgnfzKwQTvhmZoVwwjczK4QTvplZIZzwzcwK4YRv\nZlaIyhK+pB9JmidpZlXLMDOz1lXZwp8KHFRh+WZm1obKEn5E3Aw8W1X5ZmbWntr78CVNkdQjqWf+\nwrqjMTNbe9We8CPi/IiYHBGTx25adzRmZmuv2hO+mZkNDSd8M7NCVHlb5uXAH4AdJD0h6biqlmVm\nZoOr7EfMI+JvqyrbzMza5y4dM7NCOOGbmRXCCd/MrBBO+GZmhXDCNzMrhBO+mVkhnPDNzArhhG9m\nVggnfDOzQjjhm5kVwgnfzKwQlT1LZ5WM2B0m9dQdhZnZWsktfDOzQjjhm5kVwgnfzKwQTvhmZoVw\nwjczK4QTvplZIZzwzcwK4YRvZlYIJ3wzs0IoIuqO4VWSlgAP1h1HPzYDFtQdRB/dGBM4rnY5rvY4\nrtfbJiLGtjJhdz1aAR6MiMl1B9GXpJ5ui6sbYwLH1S7H1R7HtXrcpWNmVggnfDOzQnRbwj+/7gAG\n0I1xdWNM4Lja5bja47hWQ1ddtDUzs+p0WwvfzMwq4oRvZlaIrkj4kg6S9KCkhyWdUXc8vSQ9Jule\nSTMk1fZTXJJ+JGmepJkNw8ZIukHSQ/nvpl0S1zmSnsx1NkPSITXENV7STZLul3SfpJPy8FrrrElc\ntdaZpBGSbpd0d47rc3n4tpJuy/vllZLW65K4pkp6tKG+dh3KuHIMwyTdJem6/LnWumpZRNT6AoYB\nfwK2A9YD7gZ2rDuuHNtjwGZdEMe+wNuAmQ3DvgKckd+fAXy5S+I6Bzit5vraEnhbfj8a+COwY911\n1iSuWusMEDAqvx8O3Aa8A7gKOCoP/x5wYpfENRX4YM3b2CnAZcB1+XOtddXqqxta+HsCD0fEIxHx\nEnAF8P6aY+oqEXEz8Gyfwe8HLszvLwQ+MKRBMWBctYuIuRFxZ36/BHgA2Jqa66xJXLWKZGn+ODy/\nAjgAuDoPr6O+BoqrVpLGAe8DLsifRc111apuSPhbA7MbPj9BF+wEWQC/kjRd0pS6g+lji4iYm98/\nBWxRZzB9/JOke3KXz5B3NTWSNBHYjdQ67Jo66xMX1FxnuYtiBjAPuIF01r0oIl7Jk9SyX/aNKyJ6\n6+vzub6+IWn9IQ7rm8DpwIr8+Q10QV21ohsSfjfbJyLeBhwM/KOkfesOqD+RziNrb/lk3wXeBOwK\nzAW+VlcgkkYBPwFOjojnGsfVWWf9xFV7nUXE8ojYFRhHOuueNNQx9KdvXJJ2As4kxbcHMAb4l6GK\nR9KhwLyImD5Uy+ykbkj4TwLjGz6Py8NqFxFP5r/zgGtIO0K3eFrSlgD577ya4wEgIp7OO+kK4AfU\nVGeShpOS6qUR8dM8uPY66y+ubqmzHMsi4CZgL2ATSb3P26p1v2yI66DcNRYR8SLw7wxtfe0NHC7p\nMVL38wHAt+iiumqmGxL+HcCb81Xu9YCjgGtrjglJIyWN7n0PHAjMbD7XkLoWOCa/Pwb4WY2xvKo3\noWZ/Qw11lvtUfwg8EBFfbxhVa50NFFfddSZprKRN8vsNgPeSri/cBHwwT1ZHffUX16yGg7ZIfeVD\nVl8RcWZEjIuIiaRcdWNEfJia66pldV81TmfWHEK6Y+FPwFl1x5Nj2o50x9DdwH11xgVcTjrVf5nU\nP3gcqd/wN8BDwK+BMV0S18XAvcA9pAS7ZQ1x7UPqrrkHmJFfh9RdZ03iqrXOgJ2Bu/LyZwKfycO3\nA24HHgZ+DKzfJXHdmOtrJnAJ+U6eGraz/XjtLp1a66rVlx+tYGZWiG7o0jEzsyHghG9mVggnfDOz\nQjjhm5kVwgnfzKwQTvi2xpK0PD8tcaakH0vasKY4Tq5r2WbtcMK3NdkLEbFrROwEvASc0OqMkoZ1\nMI6TASd863pO+La2uAXYHkDS0fk56jMkfb83uUtaKulrku4G9pK0h6Tf5+et3y5pdH5Y11cl3ZEf\nzvUPed79JE2TdLWkWZIuVfIJYCvgJkk35Wm/K6mn8Rnuefghed7pks5reJb6yPzQtNvzM9b9tFir\nhBO+rfHyM0wOBu6V9BbgSGDvSA/dWg58OE86ErgtInYh/VfklcBJ+fN7gBdI/y28OCL2ID2c66OS\nts3z70Zqze9I+s/KvSPiPGAOsH9E7J+nOysiJpP+U/RdknaWNAL4PnBwROwOjG1YhbNI/6K/J7A/\n8NX8OA+zjlp38EnMutYG+dG5kFr4PwSmALsDd6RHrbABrz0kbTnpwWUAOwBzI+IOgMhP05R0ILCz\npN7nomwMvJnUZXR7RDyRp5sBTARu7SeuI/LjtNcl/ejJjqTG1SMR8Wie5vIcK6TnNB0u6bT8eQQw\ngfQ8G7OOccK3NdkLuRX/qvxArQsj4sx+pl8WEcsHKVPAxyPil33K3Q94sWHQcvrZf/LZwGnAHhGx\nUNJUUgIfbJn/IyIeHGQ6s9XiLh1b2/wG+KCkzeHV37Hdpp/pHgS2lLRHnm507hr6JXBifowxkv6q\nhe6VJaSfLATYCHgeWCxpC1JXU+/ytss/fAKp26nXL4GP54MVknZrdWXN2uEWvq1VIuJ+SWeTfqls\nHdKTPP8ReLzPdC9JOhL41/zo3RdI/fgXkLpq7swJeD6D/1zd+cAvJM2JiP0l3QXMIv2S2+/y8l6Q\n9LE83fOkx4L3+j+kX1G6J8f8KHDoKleC2QD8tEyzISJpVEQszQeS7wAPRcQ36o7LyuEuHbOh89F8\nsfc+0sXg79ccjxXGLXwzs0K4hW9mVggnfDOzQjjhm5kVwgnfzKwQTvhmZoX4/8uJ/yE9fWFpAAAA\nAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x114e93b90>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "values = prediction_count_dict.values()\n",
    "plot_values = (x*100/prediction_count_sum for x in values)\n",
    "plt.barh(list(prediction_count_dict.keys()), list(plot_values), color='gold')\n",
    "plt.ylabel(\"Rating\")\n",
    "plt.xlabel(\"Percentage\")\n",
    "plt.title(\"Distribution of Ratings (Predictions) - \"+str(category))\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 567,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(93026, u'Kelby Training DVD: Mastering Blend Modes in Adobe Photoshop CS5 By Corey Barker', '\"[[u\\'Electronics\\''), (165519, u'Kelby Training DVD: Adobe Photoshop CS5 Crash Course By Matt Kloskowski', '\"[[u\\'Electronics\\''), (86550, u'Digital Organizer and Messenger', '\"[[u\\'Electronics\\'')]\n",
      "[(849668, 93026, 5.0), (263499, 165519, 5.0), (1266909, 86550, 1.0)]\n"
     ]
    }
   ],
   "source": [
    "product_titles = product_meta_data.map(lambda x: str(x[1]))\n",
    "# print(product_titles.count())\n",
    "print(product_meta_data.take(3))\n",
    "# print(product_titles.take(3))\n",
    "print(ratings_data.take(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 568,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(131072, 2), (393216, 1), (0, 5), (131080, 2), (16, 15)]\n"
     ]
    }
   ],
   "source": [
    "def get_counts_and_averages(ID_and_ratings_tuple):\n",
    "    nratings = len(ID_and_ratings_tuple[1])\n",
    "    return ID_and_ratings_tuple[0], (nratings, float(sum(x for x in ID_and_ratings_tuple[1]))/nratings)\n",
    "\n",
    "product_id_ratings = (ratings_data.map(lambda x: (int(x[1]), float(x[2]))).groupByKey())\n",
    "# print(product_id_ratings.take(3))\n",
    "product_id_rating_count_avg = product_id_ratings.map(get_counts_and_averages)\n",
    "# [0] = prodId, [1] = ([0] = count of ratings, [1] = avg of ratings)\n",
    "product_rating_counts = product_id_rating_count_avg.map(lambda x: (x[0], x[1][0]))\n",
    "print(product_rating_counts.take(5))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 741,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "19\n",
      "[(10000, 6466, 4.0), (10000, 285558, 4.0), (10000, 123942, 4.0)]\n",
      "[(146110, (4.0, u'CowboyStudio Professional Neoprene Neck Strap Neckstrap for NIKON Camera')), (314880, (5.0, u'\"Wacom Intuos3 4x5 Pen Tablet')), (318991, (5.0, u'Klipsch RCX4 Reference Center Channel Speaker'))]\n"
     ]
    }
   ],
   "source": [
    "target_user_id = 10000\n",
    "target_user_ratings = ratings_data.filter(lambda x: int(x[0]) == target_user_id)\n",
    "print(target_user_ratings.count())\n",
    "print(target_user_ratings.take(3))\n",
    "target_user_ratings_ids = target_user_ratings.map(lambda x: x[1]) \n",
    "target_user_ids_title = target_user_ratings.map(lambda x: (x[1],x[2])).join(product_meta_data)\n",
    "# print(target_user_ratings_ids.take(3))\n",
    "print(target_user_ids_title.take(3))\n",
    "target_user_unrated_products = (product_meta_data.map(lambda x: (target_user_id, x[0])))\n",
    "# print(target_user_unrated_products.take(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 742,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_user_predictions = model.predictAll(target_user_unrated_products)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 743,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "427097\n",
      "332580\n",
      "[Rating(user=10000, product=257320, rating=3.1137452317653573), Rating(user=10000, product=23776, rating=1.2625907278107662), Rating(user=10000, product=307248, rating=1.760473646492331)]\n"
     ]
    }
   ],
   "source": [
    "print(product_meta_data.count())\n",
    "print(target_user_predictions.count())\n",
    "print(target_user_predictions.take(3))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Getting top recommendations\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 744,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "332580\n",
      "[(0, ((3.331943515026759, u'DROK DC Converter 12V to 5V 3A Car USB Charger Power Supply for iPhone iPad Phone In Vehicle DVR Recorder'), 5)), (131076, ((0.9299691689928007, u'Pioneer DEH-6010MP In-Dash Single DIN MP3/WMA/WAV/CD Tuner AM/FM with Rotary Commander and 2 RCA pre-outs AUX &amp; Remote'), 3)), (262152, ((2.6734830248002517, u'4-Port USB Peripheral Sharing Device'), 1))]\n"
     ]
    }
   ],
   "source": [
    "target_user_predictions_2 = target_user_predictions.map(lambda x: (x.product, x.rating))\n",
    "\n",
    "target_user_predictions_titles_counts = target_user_predictions_2.join(product_meta_data).join(product_rating_counts)\n",
    "print(target_user_predictions_titles_counts.count())\n",
    "print(target_user_predictions_titles_counts.take(3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 745,
   "metadata": {},
   "outputs": [],
   "source": [
    "target_user_predictions_titles_counts = target_user_predictions_titles_counts.map(lambda r: (r[1][0][1], r[1][0][0], r[1][1], r[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 746,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_product_recommendations = target_user_predictions_titles_counts.filter(lambda r: r[2]>=25).takeOrdered(10, key=lambda x: -x[1])\n",
    "user_rated_products = target_user_ids_title.takeOrdered(20, key=lambda x: -x[1][0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 747,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>314880</td>\n",
       "      <td>(5.0, \"Wacom Intuos3 4x5 Pen Tablet)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>318991</td>\n",
       "      <td>(5.0, Klipsch RCX4 Reference Center Channel Speaker)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>259121</td>\n",
       "      <td>(5.0, Tamrac 3536 Express 6 Camera Bag -Black)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>272012</td>\n",
       "      <td>(5.0, \"SanDisk Ultra 64GB MicroSDXC Class 10 UHS Memory Card Speed Up To 30MB/s With Adapter)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>220222</td>\n",
       "      <td>(5.0, Klipsch RB-41 II Bookshelf Speaker - Black - Pair)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>235673</td>\n",
       "      <td>(5.0, DVI to HDMI Cable 6ft Male-Male)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>146110</td>\n",
       "      <td>(4.0, CowboyStudio Professional Neoprene Neck Strap Neckstrap for NIKON Camera)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>291542</td>\n",
       "      <td>(4.0, Apple 4 GB iPod nano - White (1G)  (Discontinued by Manufacturer))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>123942</td>\n",
       "      <td>(4.0, Philips MCM530 150-Watt Micro Shelf System with 5-Disc CD/MP3 Changer and Digital AM/FM Tuner (Discontinued by Manufacturer))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>47694</td>\n",
       "      <td>(4.0, Genius Wireless Energy Mouse with Built-In 2700mAh Powerbank for iOS and Android Smartphone Devices)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>391454</td>\n",
       "      <td>(4.0, Polk Audio TSx 220B Bookshelf Speaker - Cherry)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>289485</td>\n",
       "      <td>(4.0, \"SanDisk SDDR-103 MobileMate SD+ 5-in-1 Mobile Reader (SD/ MiniSD/ MMC/ RS-MMC/ TransFlash)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>6466</td>\n",
       "      <td>(4.0, \"Tiffen 49mm Photo Essentials Kit with UV Protector)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>285558</td>\n",
       "      <td>(4.0, Logitech Optical Marble Mouse (USB/PS2))</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>80455</td>\n",
       "      <td>(3.0, Griffin iTrip Wireless Mobile FM Transmitter to Car Radio for iPod nano 1G)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>57582</td>\n",
       "      <td>(2.0, DVI Gear HDMI Cable 2M 6 feet)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>139247</td>\n",
       "      <td>(2.0, \"Arion Legacy AR306-BK 2.1 Speaker System with Subwoofer for MP3)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>201398</td>\n",
       "      <td>(2.0, Denon AVR-1312 5.1 Channel A/V Home Theater Receiver)</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>153797</td>\n",
       "      <td>(1.0, Apple USB Power Adapter for iPod (White))</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "         0  \\\n",
       "0   314880   \n",
       "1   318991   \n",
       "2   259121   \n",
       "3   272012   \n",
       "4   220222   \n",
       "5   235673   \n",
       "6   146110   \n",
       "7   291542   \n",
       "8   123942   \n",
       "9   47694    \n",
       "10  391454   \n",
       "11  289485   \n",
       "12  6466     \n",
       "13  285558   \n",
       "14  80455    \n",
       "15  57582    \n",
       "16  139247   \n",
       "17  201398   \n",
       "18  153797   \n",
       "\n",
       "                                                                                                                                      1  \n",
       "0   (5.0, \"Wacom Intuos3 4x5 Pen Tablet)                                                                                                 \n",
       "1   (5.0, Klipsch RCX4 Reference Center Channel Speaker)                                                                                 \n",
       "2   (5.0, Tamrac 3536 Express 6 Camera Bag -Black)                                                                                       \n",
       "3   (5.0, \"SanDisk Ultra 64GB MicroSDXC Class 10 UHS Memory Card Speed Up To 30MB/s With Adapter)                                        \n",
       "4   (5.0, Klipsch RB-41 II Bookshelf Speaker - Black - Pair)                                                                             \n",
       "5   (5.0, DVI to HDMI Cable 6ft Male-Male)                                                                                               \n",
       "6   (4.0, CowboyStudio Professional Neoprene Neck Strap Neckstrap for NIKON Camera)                                                      \n",
       "7   (4.0, Apple 4 GB iPod nano - White (1G)  (Discontinued by Manufacturer))                                                             \n",
       "8   (4.0, Philips MCM530 150-Watt Micro Shelf System with 5-Disc CD/MP3 Changer and Digital AM/FM Tuner (Discontinued by Manufacturer))  \n",
       "9   (4.0, Genius Wireless Energy Mouse with Built-In 2700mAh Powerbank for iOS and Android Smartphone Devices)                           \n",
       "10  (4.0, Polk Audio TSx 220B Bookshelf Speaker - Cherry)                                                                                \n",
       "11  (4.0, \"SanDisk SDDR-103 MobileMate SD+ 5-in-1 Mobile Reader (SD/ MiniSD/ MMC/ RS-MMC/ TransFlash)                                    \n",
       "12  (4.0, \"Tiffen 49mm Photo Essentials Kit with UV Protector)                                                                           \n",
       "13  (4.0, Logitech Optical Marble Mouse (USB/PS2))                                                                                       \n",
       "14  (3.0, Griffin iTrip Wireless Mobile FM Transmitter to Car Radio for iPod nano 1G)                                                    \n",
       "15  (2.0, DVI Gear HDMI Cable 2M 6 feet)                                                                                                 \n",
       "16  (2.0, \"Arion Legacy AR306-BK 2.1 Speaker System with Subwoofer for MP3)                                                              \n",
       "17  (2.0, Denon AVR-1312 5.1 Channel A/V Home Theater Receiver)                                                                          \n",
       "18  (1.0, Apple USB Power Adapter for iPod (White))                                                                                      "
      ]
     },
     "execution_count": 747,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.set_option('display.max_colwidth', -1)\n",
    "pd.DataFrame(user_rated_products)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 748,
   "metadata": {
    "scrolled": false
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Pentax K2000 10.2MP Digital SLR Camera with 18-55mm f/3.5-5.6 DA L Lens and AF200FG Flash</td>\n",
       "      <td>4.018570</td>\n",
       "      <td>25</td>\n",
       "      <td>211074</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>JAVOedge Cherry Blossom Flip Style Case for the Barnes &amp;amp; Noble Nook (Ocean Blue) - First Generation</td>\n",
       "      <td>3.980742</td>\n",
       "      <td>26</td>\n",
       "      <td>273396</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Panasonic PV-GS500 4MP 3CCD MiniDV Camcorder with 12x Optical Image Stabilized Zoom</td>\n",
       "      <td>3.978618</td>\n",
       "      <td>48</td>\n",
       "      <td>292941</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Olympus EX-25mm Macro Extension Tube for Olympus Digital SLR Cameras</td>\n",
       "      <td>3.944165</td>\n",
       "      <td>32</td>\n",
       "      <td>408677</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>\"EasyAcc &amp;reg; Portable Adjust Stand Docks Holder Docking Station for all 4-10 inch Smart phone/E-reader/Tablet:iPad</td>\n",
       "      <td>3.939665</td>\n",
       "      <td>74</td>\n",
       "      <td>21806</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>Olympus VF-4 ViewFinder (Black)</td>\n",
       "      <td>3.936197</td>\n",
       "      <td>25</td>\n",
       "      <td>105768</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>\"SMDV RFN-4s Slim Wireless Remote Shutter Cable Release</td>\n",
       "      <td>3.933766</td>\n",
       "      <td>49</td>\n",
       "      <td>426807</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td></td>\n",
       "      <td>3.933062</td>\n",
       "      <td>26</td>\n",
       "      <td>202423</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>\"Pioneer AVH-P6300BT 7-Inch In-Dash DVD A/V Receiver with iPod/iPhone Control</td>\n",
       "      <td>3.930538</td>\n",
       "      <td>41</td>\n",
       "      <td>425920</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>Coolerguys 120x120x25mm USB Fan with Grill</td>\n",
       "      <td>3.926401</td>\n",
       "      <td>82</td>\n",
       "      <td>185599</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                      0  \\\n",
       "0  Pentax K2000 10.2MP Digital SLR Camera with 18-55mm f/3.5-5.6 DA L Lens and AF200FG Flash                              \n",
       "1  JAVOedge Cherry Blossom Flip Style Case for the Barnes &amp; Noble Nook (Ocean Blue) - First Generation                \n",
       "2  Panasonic PV-GS500 4MP 3CCD MiniDV Camcorder with 12x Optical Image Stabilized Zoom                                    \n",
       "3  Olympus EX-25mm Macro Extension Tube for Olympus Digital SLR Cameras                                                   \n",
       "4  \"EasyAcc &reg; Portable Adjust Stand Docks Holder Docking Station for all 4-10 inch Smart phone/E-reader/Tablet:iPad   \n",
       "5  Olympus VF-4 ViewFinder (Black)                                                                                        \n",
       "6  \"SMDV RFN-4s Slim Wireless Remote Shutter Cable Release                                                                \n",
       "7                                                                                                                         \n",
       "8  \"Pioneer AVH-P6300BT 7-Inch In-Dash DVD A/V Receiver with iPod/iPhone Control                                          \n",
       "9  Coolerguys 120x120x25mm USB Fan with Grill                                                                             \n",
       "\n",
       "          1   2       3  \n",
       "0  4.018570  25  211074  \n",
       "1  3.980742  26  273396  \n",
       "2  3.978618  48  292941  \n",
       "3  3.944165  32  408677  \n",
       "4  3.939665  74  21806   \n",
       "5  3.936197  25  105768  \n",
       "6  3.933766  49  426807  \n",
       "7  3.933062  26  202423  \n",
       "8  3.930538  41  425920  \n",
       "9  3.926401  82  185599  "
      ]
     },
     "execution_count": 748,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.DataFrame(top_product_recommendations)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### SAVE THE MODEL"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 526,
   "metadata": {},
   "outputs": [],
   "source": [
    "from pyspark.mllib.recommendation import MatrixFactorizationModel\n",
    "\n",
    "model_name = \"model_\"+str(category)+\"_\"+str(best_rank)\n",
    "model_path = os.path.join(models_path, model_name)\n",
    "\n",
    "model.save(sc, model_path)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 527,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/05/30 00:50:41 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Found 5 items\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-30 00:23 /models/model_Books_8\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-29 20:16 /models/model_Electronics_8\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-29 00:56 /models/model_Musical_Instruments_64\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-26 20:36 /models/model_Musical_Instruments_8\n",
      "drwxr-xr-x   - rahul supergroup          0 2018-05-26 16:08 /models/model_Cell_Phones_and_Accessories_8\n"
     ]
    }
   ],
   "source": [
    "! hdfs dfs -ls -t /models"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "##### DELETE EXISTING MODELS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_path"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 524,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "18/05/30 00:22:58 WARN util.NativeCodeLoader: Unable to load native-hadoop library for your platform... using builtin-java classes where applicable\n",
      "Deleted hdfs://192.168.1.6:9000/models/model_Books_8\n"
     ]
    }
   ],
   "source": [
    "# hdfs_cmd=\"hdfs dfs -rm -r -f \"+str(model_path)\n",
    "# ! $hdfs_cmd"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### END OF NOTEBOOK"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}