weak_recsys_douban.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Weakly Supervised Recommendation Systems"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Experiments steps:\n",
    " 1. **User's Preferences Model**: Leverage the most *explicit* ratings to build a *rate/rank prediction model*. This is a simple *Explicit Matrix Factorization* model. \n",
    " 2. **Generate Weak DataSet**: Use the above model to *predict* for all user/item pairs $(u,i)$ in *implicit feedback dataset* to build a new *weak explicit dataset* $(u, i, r^*)$.\n",
    " 3. **Evaluate**: Use the intact test split in the most explicit feedback, in order to evaluate the performance of any model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Explicit Model Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section contains all the experiments based on the explicit matrix factorization model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explicit Rate Model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 4,960 interactions of 3,886 users and 21,833 items\n",
      "Explicit dataset (VALID) contains 4,960 interactions of 3,886 users and 21,833 items\n",
      "Explicit dataset (TRAIN) contains 39,678 interactions of 3,886 users and 21,833 items\n",
      "Implicit dataset (READ/READING/TAG/COMMENT) contains 428,597 interactions of 3,886 users and 21,833 items\n",
      "Implicit dataset (WISH) contains 496,204 interactions of 3,886 users and 21,833 items\n",
      "--------------------\n",
      "RMSE: 0.3104\n",
      "MRR: 0.0034\n",
      "nDCG: 0.0059\n",
      "nDCG@10: 0.0022\n",
      "nDCG@5: 0.0009\n",
      "MAP: 0.0018\n",
      "success@10: 0.0095\n",
      "success@5: 0.0027\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_wish = utils.parse_douban()\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ/READING/TAG/COMMENT) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (WISH) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_wish.ratings), ','),\n",
    "          format(dataset_wish.num_users, ','),\n",
    "          format(dataset_wish.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_explicit(train_interactions=dataset_recommend_train, \n",
    "                             valid_interactions=dataset_recommend_dev,\n",
    "                             run_name='model_douban_explicit_rate')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Remove valid/test ratings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "test_interact = set()\n",
    "for (uid, iid) in zip(dataset_recommend_test.user_ids, dataset_recommend_test.item_ids):\n",
    "    test_interact.add((uid, iid))\n",
    "\n",
    "for (uid, iid) in zip(dataset_recommend_dev.user_ids, dataset_recommend_dev.item_ids):\n",
    "    test_interact.add((uid, iid))\n",
    "\n",
    "# clean implicit dataset from test/dev rating\n",
    "for idx, (uid, iid, r) in enumerate(zip(dataset_read.user_ids, dataset_read.item_ids, dataset_read.ratings)):\n",
    "    if (uid, iid) in test_interact:\n",
    "        dataset_read.ratings[idx] = -1"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Explicit Read/Reading/Tag/Comment Model"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Leverage the **explicit rate model** trained at the previous section to annotate **missing values** in the **read/reading/tag/comment** dataset."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch 1 start at:  Tue Apr 23 09:07:20 2019\n",
      "epoch 1 end at:  Tue Apr 23 09:07:23 2019\n",
      "RMSE: 0.3986\n",
      "epoch 2 start at:  Tue Apr 23 09:07:24 2019\n",
      "epoch 2 end at:  Tue Apr 23 09:07:26 2019\n",
      "RMSE: 0.3733\n",
      "epoch 3 start at:  Tue Apr 23 09:07:27 2019\n",
      "epoch 3 end at:  Tue Apr 23 09:07:29 2019\n",
      "RMSE: 0.3525\n",
      "epoch 4 start at:  Tue Apr 23 09:07:30 2019\n",
      "epoch 4 end at:  Tue Apr 23 09:07:32 2019\n",
      "RMSE: 0.3427\n",
      "epoch 5 start at:  Tue Apr 23 09:07:33 2019\n",
      "epoch 5 end at:  Tue Apr 23 09:07:36 2019\n",
      "RMSE: 0.3348\n",
      "epoch 6 start at:  Tue Apr 23 09:07:36 2019\n",
      "epoch 6 end at:  Tue Apr 23 09:07:39 2019\n",
      "RMSE: 0.3357\n",
      "--------------------\n",
      "RMSE: 0.3271\n",
      "MRR: 0.0223\n",
      "nDCG: 0.0345\n",
      "nDCG@10: 0.0207\n",
      "nDCG@5: 0.0120\n",
      "MAP: 0.0149\n",
      "success@10: 0.0684\n",
      "success@5: 0.0302\n"
     ]
    }
   ],
   "source": [
    "# annotate the missing values in the play dataset based on the explicit recommend model\n",
    "dataset_read = utils.annotate(interactions=dataset_read, \n",
    "                              model=model, \n",
    "                              run_name='dataset_douban_read_explicit_annotated')\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_explicit(train_interactions=dataset_read, \n",
    "                             valid_interactions=dataset_recommend_dev,\n",
    "                             run_name='model_douban_explicit_read')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Implicit Model Experiments"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This section contains all the experiments based on the implicit matrix factorization model."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Implicit Model using Negative Sampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 4,960 interactions of 3,886 users and 21,833 items\n",
      "Explicit dataset (VALID) contains 4,960 interactions of 3,886 users and 21,833 items\n",
      "Explicit dataset (TRAIN) contains 39,678 interactions of 3,886 users and 21,833 items\n",
      "Implicit dataset (READ/READING/TAG/COMMENT) contains 428,597 interactions of 3,886 users and 21,833 items\n",
      "Implicit dataset (WISH) contains 496,204 interactions of 3,886 users and 21,833 items\n",
      "epoch 1 start at:  Sat Apr 20 11:12:30 2019\n",
      "epoch 1 end at:  Sat Apr 20 11:12:33 2019\n",
      "MRR: 0.0247\n",
      "epoch 2 start at:  Sat Apr 20 11:12:45 2019\n",
      "epoch 2 end at:  Sat Apr 20 11:12:48 2019\n",
      "MRR: 0.0181\n",
      "--------------------\n",
      "RMSE: 4.2549\n",
      "MRR: 0.0200\n",
      "nDCG: 0.0239\n",
      "nDCG@10: 0.0178\n",
      "nDCG@5: 0.0127\n",
      "MAP: 0.0119\n",
      "success@10: 0.0543\n",
      "success@5: 0.0317\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_wish = utils.parse_douban()\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ/READING/TAG/COMMENT) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (WISH) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_wish.ratings), ','),\n",
    "          format(dataset_wish.num_users, ','),\n",
    "          format(dataset_wish.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = utils.train_implicit_negative_sampling(train_interactions=dataset_read, \n",
    "                                               valid_interactions=dataset_recommend_dev,\n",
    "                                               run_name='model_douban_implicit_read2')\n",
    "\n",
    "# evaluate the new model\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "rmse = rmse_score(model=model, test=dataset_recommend_test)\n",
    "print('-'*20)\n",
    "print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Popularity"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Explicit dataset (TEST) contains 4,960 interactions of 3,886 users and 21,833 items\n",
      "Explicit dataset (VALID) contains 4,960 interactions of 3,886 users and 21,833 items\n",
      "Explicit dataset (TRAIN) contains 39,678 interactions of 3,886 users and 21,833 items\n",
      "Implicit dataset (READ/READING/TAG/COMMENT) contains 428,597 interactions of 3,886 users and 21,833 items\n",
      "Implicit dataset (WISH) contains 496,204 interactions of 3,886 users and 21,833 items\n",
      "fit the model\n",
      "evaluate the model\n",
      "MRR: 0.0215\n",
      "nDCG: 0.0253\n",
      "nDCG@10: 0.0191\n",
      "nDCG@5: 0.0147\n",
      "MAP: 0.0134\n",
      "success@10: 0.0550\n",
      "success@5: 0.0344\n"
     ]
    }
   ],
   "source": [
    "import utils\n",
    "from popularity import PopularityModel\n",
    "from spotlight.evaluation import rmse_score\n",
    "\n",
    "dataset_recommend_train, dataset_recommend_test, dataset_recommend_dev, dataset_read, dataset_wish = utils.parse_douban()\n",
    "\n",
    "print('Explicit dataset (TEST) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_test.ratings), ','),\n",
    "          format(dataset_recommend_test.num_users, ','),\n",
    "          format(dataset_recommend_test.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (VALID) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_dev.ratings), ','),\n",
    "          format(dataset_recommend_dev.num_users, ','),\n",
    "          format(dataset_recommend_dev.num_items, ',')))\n",
    "\n",
    "print('Explicit dataset (TRAIN) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_recommend_train.ratings), ','),\n",
    "          format(dataset_recommend_train.num_users, ','),\n",
    "          format(dataset_recommend_train.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (READ/READING/TAG/COMMENT) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_read.ratings), ','),\n",
    "          format(dataset_read.num_users, ','),\n",
    "          format(dataset_read.num_items, ',')))\n",
    "\n",
    "print('Implicit dataset (WISH) contains %s interactions of %s users and %s items'%(\n",
    "          format(len(dataset_wish.ratings), ','),\n",
    "          format(dataset_wish.num_users, ','),\n",
    "          format(dataset_wish.num_items, ',')))\n",
    "\n",
    "# train the explicit model based on recommend feedback\n",
    "model = PopularityModel()\n",
    "print('fit the model')\n",
    "model.fit(interactions=dataset_recommend_train)\n",
    "\n",
    "# evaluate the new model\n",
    "print('evaluate the model')\n",
    "mrr, ndcg, ndcg10, ndcg_5, mmap, success_10, success_5 = utils.evaluate(interactions=dataset_recommend_test,\n",
    "                                                                        model=model,\n",
    "                                                                        topk=20)\n",
    "# rmse = rmse_score(model=model, test=dataset_recommend_test, batch_size=512)\n",
    "# print('-'*20)\n",
    "# print('RMSE: {:.4f}'.format(rmse))\n",
    "print('MRR: {:.4f}'.format(mrr))\n",
    "print('nDCG: {:.4f}'.format(ndcg))\n",
    "print('nDCG@10: {:.4f}'.format(ndcg10))\n",
    "print('nDCG@5: {:.4f}'.format(ndcg_5))\n",
    "print('MAP: {:.4f}'.format(mmap))\n",
    "print('success@10: {:.4f}'.format(success_10))\n",
    "print('success@5: {:.4f}'.format(success_5))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}