product_ratings.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import gzip\n",
    "import numpy\n",
    "import random\n",
    "from collections import defaultdict\n",
    "\n",
    "def readGz(f):\n",
    "  for l in gzip.open(f):\n",
    "    yield eval(l)\n",
    "### Rating baseline: compute averages for each user, or return the global average if we've never seen the user before\n",
    "allRatings = []\n",
    "userRatings = defaultdict(list)\n",
    "data = list(readGz(\"Assignment1/HW3/assignment1/train.json.gz\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# initialzation\n",
    "users_perItem = defaultdict(list)\n",
    "items_perUser = defaultdict(list)\n",
    "userItem_rating = {} # rating for a specific user with respect to a specific item\n",
    "user_bias = {}\n",
    "item_bias = {}\n",
    "alpha = 0\n",
    "lam = 6\n",
    "\n",
    "prev_MSE = 20 #randomly initializing previous MSE\n",
    "MSE = 10 #randomly intitializing current MSE\n",
    "\n",
    "# store the parameters that result in smallest MSE\n",
    "user_bias_final = {}\n",
    "item_bias_final = {}\n",
    "alpha_final = 0\n",
    "\n",
    "for l in data:#[:100000]:\n",
    "    user,item = l['reviewerID'],l['itemID']\n",
    "    users_perItem[item].append(l['reviewerID'])\n",
    "    items_perUser[user].append(l['itemID'])\n",
    "    userItem_rating[user+item] = l['rating']\n",
    "    user_bias[user] = 0\n",
    "    item_bias[item] = 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "prev_MSE = 10 #randomly initializing previous MSE\n",
    "MSE = 9 #randomly intitializing current MSE\n",
    "while MSE < prev_MSE:#abs(prev_MSE - MSE) > 0.00001:\n",
    "    prev_MSE = MSE\n",
    "    ## update alpha\n",
    "    alpha = [l['rating'] - (user_bias[l['reviewerID']] + item_bias[l['itemID']]) for l in data]#[:100000]]\n",
    "    alpha = sum(alpha) / 200000#100000\n",
    "    \n",
    "    ## update user bias\n",
    "    for u in user_bias:\n",
    "        temp = sum([userItem_rating[u+i] - (alpha + item_bias[i]) for i in items_perUser[u]])\n",
    "        user_bias[u] = temp / (lam + len(items_perUser[u]))\n",
    "    \n",
    "    ## update item bias\n",
    "    for i in item_bias:\n",
    "        temp = sum([userItem_rating[u+i] - (alpha + user_bias[u]) for u in users_perItem[i]])\n",
    "        item_bias[i] = temp / (lam + len(users_perItem[i]))\n",
    "        \n",
    "    ## compute MSE\n",
    "    diff = [(l['rating'] - (alpha + user_bias[l['reviewerID']] + item_bias[l['itemID']]))**2 for l in data]#[:100000]]\n",
    "    MSE = sum(diff) / len(diff)\n",
    "\n",
    "    \n",
    "    if MSE < prev_MSE:\n",
    "        ## update final results\n",
    "        for u in user_bias:\n",
    "            user_bias_final[u] = user_bias[u]\n",
    "        for i in item_bias:\n",
    "            item_bias_final[i] = item_bias[i]\n",
    "        alpha_final = alpha"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "MSE =  0.816191926778 for validating set (lambda = 6)\n"
     ]
    }
   ],
   "source": [
    "## Validation MSE\n",
    "diff = []\n",
    "for l in data[100000:200000]:\n",
    "    if l['reviewerID'] not in user_bias and l['itemID'] not in item_bias:\n",
    "        diff.append((l['rating'] - alpha)**2)\n",
    "    elif l['reviewerID'] not in user_bias:\n",
    "        diff.append((l['rating'] - (alpha + item_bias[l['itemID']]))**2)\n",
    "    elif l['itemID'] not in item_bias:\n",
    "        diff.append((l['rating'] - (alpha + user_bias[l['reviewerID']]))**2)\n",
    "    else:\n",
    "        diff.append((l['rating'] - (alpha + user_bias[l['reviewerID']] + item_bias[l['itemID']]))**2)\n",
    "MSE = sum(diff) / len(diff)\n",
    "\n",
    "print 'MSE = ', MSE,'for validating set (lambda = 6)'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "userItemRating = []\n",
    "\n",
    "predictions = open(\"Assignment1/HW3/assignment1/predictions_Rating.txt\", 'w')\n",
    "for l in open(\"Assignment1/HW3/assignment1/pairs_Rating.txt\"):\n",
    "    if l.startswith(\"userID\"):\n",
    "        #header\n",
    "        predictions.write(l)\n",
    "        continue\n",
    "    u, i = l.strip().split('-')\n",
    "    if u not in user_bias_final and i not in item_bias_final:\n",
    "        predicted_rating = alpha_final\n",
    "    elif u not in user_bias_final:\n",
    "        predicted_rating = alpha_final + item_bias_final[i]\n",
    "    elif i not in item_bias_final:\n",
    "        predicted_rating = alpha_final + user_bias_final[u]\n",
    "    else:\n",
    "        predicted_rating = alpha_final + user_bias_final[u] + item_bias_final[i]# + gamma_inner\n",
    "        \n",
    "    if predicted_rating > 5.0:\n",
    "        predictions.write(u + '-' + i + ',' + str(5.0) + '\\n')\n",
    "    else:\n",
    "        predictions.write(u + '-' + i + ',' + str(predicted_rating) + '\\n')\n",
    "            \n",
    "predictions.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    ""
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2.0
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}