weak_recsys_goodreads_Fantasy.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weakly Supervised Recommendation Systems"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiments steps:\n",
    " 1. **User's Preferences Model**: Leverage the most *explicit* ratings to build a *rate/rank prediction model*. This is a simple *Explicit Matrix Factorization* model. \n",
    " 2. **Generate Weak DataSet**: Use the above model to *predict* for all user/item pairs $(u,i)$ in *implicit feedback dataset* to build a new *weak explicit dataset* $(u, i, r^*)$.\n",
    " 3. **Evaluate**: Use the intact test split in the most explicit feedback, in order to evaluate the performance of any model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explicit Model Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section contains all the experiments based on the explicit matrix factorization model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explicit Rate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 2,539,491 interactions of 419,774 users and 258,037 items\n",
      "Explicit dataset (VALID) contains 2,539,492 interactions of 419,774 users and 258,037 items\n",
      "Explicit dataset (TRAIN) contains 20,315,929 interactions of 419,774 users and 258,037 items\n",
      "Implicit dataset (READ) contains 26,919,068 interactions of 419,774 users and 258,037 items\n",
      "Implicit dataset (SHELVE) contains 52,141,533 interactions of 419,774 users and 258,037 items\n",
      "epoch 1 start at:  Tue Apr 23 14:40:05 2019\n",
      "epoch 1 end at:  Tue Apr 23 15:00:51 2019\n",
      "RMSE: 0.4113\n",
      "epoch 2 start at:  Tue Apr 23 15:01:02 2019\n",
      "epoch 2 end at:  Tue Apr 23 15:21:45 2019\n",
      "RMSE: 0.4089\n",
      "epoch 3 start at:  Tue Apr 23 15:21:57 2019\n",
      "epoch 3 end at:  Tue Apr 23 15:42:42 2019\n",
      "RMSE: 0.4082\n",
      "epoch 4 start at:  Tue Apr 23 15:42:54 2019\n",
      "epoch 4 end at:  Tue Apr 23 16:03:39 2019\n",
      "RMSE: 0.4079\n",
      "epoch 5 start at:  Tue Apr 23 16:03:51 2019\n",
      "epoch 5 end at:  Tue Apr 23 16:21:25 2019\n",
      "RMSE: 0.4076\n",
      "epoch 6 start at:  Tue Apr 23 16:21:37 2019\n",
      "epoch 6 end at:  Tue Apr 23 16:39:18 2019\n",
      "RMSE: 0.4076\n",
      "epoch 7 start at:  Tue Apr 23 16:39:31 2019\n",
      "epoch 7 end at:  Tue Apr 23 16:58:33 2019\n",
      "RMSE: 0.4075\n",
      "epoch 8 start at:  Tue Apr 23 16:58:47 2019\n",
      "epoch 8 end at:  Tue Apr 23 17:17:50 2019\n",
      "RMSE: 0.4074\n",
      "epoch 9 start at:  Tue Apr 23 17:18:02 2019\n",
      "epoch 9 end at:  Tue Apr 23 17:37:04 2019\n",
      "RMSE: 0.4074\n",
      "--------------------\n",
      "RMSE: 0.4077\n",
      "MRR: 0.0258\n",
      "nDCG: 0.0230\n",
      "nDCG@10: 0.0197\n",
      "nDCG@5: 0.0146\n",
      "MAP: 0.0106\n",
      "success@10: 0.0802\n",
      "success@5: 0.0496\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_shelve = utils.parse_goodreads(\n",
    "    path='/local/terrier/Collections/Recommendations/Goodreads/goodreads_interactions_fantasy_paranormal.json.gz')\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (SHELVE) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_shelve.ratings), ','),\n",
    "          format(dataset_shelve.num_users, ','),\n",
    "          format(dataset_shelve.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_explicit(train_interactions=dataset_recommend_train, \n",
    "                             valid_interactions=dataset_recommend_dev,\n",
    "                             run_name='model_goodreads_fantasy_explicit_rate')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove all valid/test samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_interact = set()\n",
    "for (uid, iid) in zip(dataset_recommend_test.user_ids, dataset_recommend_test.item_ids):\n",
    "    test_interact.add((uid, iid))\n",
    "\n",
    "for (uid, iid) in zip(dataset_recommend_dev.user_ids, dataset_recommend_dev.item_ids):\n",
    "    test_interact.add((uid, iid))\n",
    "\n",
    "# clean implicit dataset from test/dev rating\n",
    "for idx, (uid, iid, r) in enumerate(zip(dataset_read.user_ids, dataset_read.item_ids, dataset_read.ratings)):\n",
    "    if (uid, iid) in test_interact:\n",
    "        dataset_read.ratings[idx] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explicit Read Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Leverage the **explicit rate model** trained at the previous section to annotate **missing values** in the **read** dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 start at:  Wed Apr 24 17:33:55 2019\n",
      "epoch 1 end at:  Wed Apr 24 17:54:50 2019\n",
      "RMSE: 0.4156\n",
      "epoch 2 start at:  Wed Apr 24 17:55:02 2019\n",
      "epoch 2 end at:  Wed Apr 24 18:15:59 2019\n",
      "RMSE: 0.4144\n",
      "epoch 3 start at:  Wed Apr 24 18:16:11 2019\n",
      "epoch 3 end at:  Wed Apr 24 18:37:09 2019\n",
      "RMSE: 0.4141\n",
      "epoch 4 start at:  Wed Apr 24 18:37:21 2019\n",
      "epoch 4 end at:  Wed Apr 24 18:58:17 2019\n",
      "RMSE: 0.4139\n",
      "epoch 5 start at:  Wed Apr 24 18:58:29 2019\n",
      "epoch 5 end at:  Wed Apr 24 19:19:17 2019\n",
      "RMSE: 0.4139\n",
      "epoch 6 start at:  Wed Apr 24 19:19:29 2019\n",
      "epoch 6 end at:  Wed Apr 24 19:40:22 2019\n",
      "RMSE: 0.4139\n",
      "epoch 7 start at:  Wed Apr 24 19:40:34 2019\n",
      "epoch 7 end at:  Wed Apr 24 20:01:19 2019\n",
      "RMSE: 0.4139\n",
      "--------------------\n",
      "RMSE: 0.4141\n",
      "MRR: 0.0247\n",
      "nDCG: 0.0208\n",
      "nDCG@10: 0.0170\n",
      "nDCG@5: 0.0130\n",
      "MAP: 0.0101\n",
      "success@10: 0.0661\n",
      "success@5: 0.0396\n"
     ]
    }
   ],
   "source": [
    "# annotate the missing values in the play dataset based on the explicit recommend model\n",
    "dataset_read = utils.annotate(interactions=dataset_read, \n",
    "                              model=model, \n",
    "                              run_name='dataset_goodreads_fantasy_read_explicit_annotated')\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_explicit(train_interactions=dataset_read, \n",
    "                             valid_interactions=dataset_recommend_dev,\n",
    "                             run_name='model_goodreads_fantasy_explicit_read')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implicit Model Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section contains all the experiments based on the implicit matrix factorization model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implicit Model using Negative Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 2,539,491 interactions of 419,774 users and 258,037 items\n",
      "Explicit dataset (VALID) contains 2,539,492 interactions of 419,774 users and 258,037 items\n",
      "Explicit dataset (TRAIN) contains 20,315,929 interactions of 419,774 users and 258,037 items\n",
      "Implicit dataset (READ) contains 26,919,068 interactions of 419,774 users and 258,037 items\n",
      "Implicit dataset (SHELVE) contains 52,141,533 interactions of 419,774 users and 258,037 items\n",
      "--------------------\n",
      "RMSE: 3.3754\n",
      "MRR: 0.1145\n",
      "nDCG: 0.0982\n",
      "nDCG@10: 0.0829\n",
      "nDCG@5: 0.0647\n",
      "MAP: 0.0528\n",
      "success@10: 0.2732\n",
      "success@5: 0.1757\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_shelve = utils.parse_goodreads(\n",
    "    path='/local/terrier/Collections/Recommendations/Goodreads/goodreads_interactions_fantasy_paranormal.json.gz')\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (SHELVE) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_shelve.ratings), ','),\n",
    "          format(dataset_shelve.num_users, ','),\n",
    "          format(dataset_shelve.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_implicit_negative_sampling(train_interactions=dataset_read, \n",
    "                                               valid_interactions=dataset_recommend_dev,\n",
    "                                               run_name='model_goodreads_fantasy_implicit')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Popularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 2,539,491 interactions of 419,774 users and 258,037 items\n",
      "Explicit dataset (VALID) contains 2,539,492 interactions of 419,774 users and 258,037 items\n",
      "Explicit dataset (TRAIN) contains 20,315,929 interactions of 419,774 users and 258,037 items\n",
      "Implicit dataset (READ) contains 26,919,068 interactions of 419,774 users and 258,037 items\n",
      "Implicit dataset (SHELVE) contains 52,141,533 interactions of 419,774 users and 258,037 items\n",
      "fit the model\n",
      "evaluate the model\n",
      "MRR: 0.1143\n",
      "nDCG: 0.0981\n",
      "nDCG@10: 0.0819\n",
      "nDCG@5: 0.0651\n",
      "MAP: 0.0527\n",
      "success@10: 0.2669\n",
      "success@5: 0.1773\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from spotlight.evaluation import rmse_score\n",
    "from popularity import PopularityModel\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_shelve = utils.parse_goodreads(\n",
    "    path='/local/terrier/Collections/Recommendations/Goodreads/goodreads_interactions_fantasy_paranormal.json.gz')\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (SHELVE) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_shelve.ratings), ','),\n",
    "          format(dataset_shelve.num_users, ','),\n",
    "          format(dataset_shelve.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = PopularityModel()\n",
    "print('fit the model')\n",
    "model.fit(interactions=dataset_recommend_train)\n",
    "\n",
    "# evaluate the new model\n",
    "print('evaluate the model')\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "# rmse = rmse_score(model=model, test=dataset_recommend_test, batch_size=512)\n",
    "# print('-'*20)\n",
    "# print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}