models.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import implicit\n",
    "import numpy as np\n",
    "from tqdm import tqdm_notebook\n",
    "import pandas as pd\n",
    "import csv \n",
    "import scipy\n",
    "from scipy.sparse import coo_matrix\n",
    "from scipy.sparse.linalg import svds\n",
    "from implicit.nearest_neighbours import bm25_weight\n",
    "from implicit import alternating_least_squares\n",
    "import umap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open('final_interactions_count') as csvfile:\n",
    "    datareader = csv.reader(csvfile, delimiter=' ')\n",
    "    for subreddit, user, comments, _ in datareader:\n",
    "        data.append([user, subreddit, int(comments)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame.from_records(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data.columns = ['user', 'subreddit', 'comments']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "data['user'] = data['user'].astype(\"category\")\n",
    "data['subreddit'] = data['subreddit'].astype(\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a sparse matrix of all the artist/user/play triples\n",
    "comments = coo_matrix((data['comments'].astype(float), \n",
    "                   (data['subreddit'].cat.codes, \n",
    "                    data['user'].cat.codes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "csr_comments = comments.tocsr()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1537767"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "csr_comments.nnz"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Latent Semantic Analysis"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "# toggle this variable if you want to recalculate the als factors\n",
    "read_als_factors_from_file = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "This method is deprecated. Please use the AlternatingLeastSquares class instead\n",
      "WARNING:root:OpenBLAS detected. Its highly recommend to set the environment variable 'export OPENBLAS_NUM_THREADS=1' to disable its internal multithreading\n",
      "100%|██████████| 15.0/15 [00:14<00:00,  1.29it/s]\n"
     ]
    }
   ],
   "source": [
    "if read_als_factors_from_file:\n",
    "    subreddit_factors = np.load('subreddit_factors_als.npy')\n",
    "    user_factors = np.load('user_factors_als.npy')\n",
    "else:\n",
    "    subreddit_factors, user_factors = alternating_least_squares(bm25_weight(comments), 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TopRelated(object):\n",
    "    def __init__(self, subreddit_factors):\n",
    "        norms = np.linalg.norm(subreddit_factors, axis=-1)\n",
    "        self.factors = subreddit_factors / norms[:, np.newaxis]\n",
    "        self.subreddits = data['subreddit'].cat.categories.array.to_numpy()\n",
    "\n",
    "    def get_related(self, subreddit, N=10):\n",
    "        subredditid = np.where(self.subreddits == subreddit)[0][0]\n",
    "        scores = self.factors.dot(self.factors[subredditid])\n",
    "        best = np.argpartition(scores, -N)[-N:]\n",
    "        best_ = [self.subreddits[i] for i in best]\n",
    "        return sorted(zip(best_, scores[best]), key=lambda x: -x[1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "top_related = TopRelated(subreddit_factors)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('OnePiece', 0.99999994),\n",
       " ('Naruto', 0.9689143),\n",
       " ('allblue', 0.96665776),\n",
       " ('bleach', 0.96390015),\n",
       " ('Feels', 0.9556872),\n",
       " ('manga', 0.94707495),\n",
       " ('Collabcomics', 0.9412527),\n",
       " ('whowouldwin', 0.9372169),\n",
       " ('anime', 0.9361908),\n",
       " ('doujinshi', 0.93373656)]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "top_related.get_related('OnePiece')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14842, 20)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddit_factors.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(14842, 2)"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddits_embedded = umap.UMAP().fit_transform(subreddit_factors)\n",
    "subreddits_embedded.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[1.0131747, 5.303012 ],\n",
       "       [2.208454 , 3.8350732],\n",
       "       [6.189175 , 1.7819154],\n",
       "       ...,\n",
       "       [6.3854785, 2.2662826],\n",
       "       [7.953953 , 0.7833559],\n",
       "       [2.5185633, 1.8170209]], dtype=float32)"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "subreddits_embedded"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "subreddits = data['subreddit'].cat.categories.array.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "import random\n",
    "\n",
    "indices = random.sample(range(len(subreddits)), 1000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "sampled_subreddits = subreddits[indices]\n",
    "sampled_subreddits_embedded = subreddits_embedded[indices]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "High five! You successfully sent some data to your account on plotly. View your plot in your browser at https://plot.ly/~abkds/0 or inside your plot.ly account where it is named 'basic-scatter'\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<iframe id=\"igraph\" scrolling=\"no\" style=\"border:none;\" seamless=\"seamless\" src=\"https://plot.ly/~abkds/0.embed\" height=\"525px\" width=\"100%\"></iframe>"
      ],
      "text/plain": [
       "<plotly.tools.PlotlyDisplay object>"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import plotly\n",
    "import plotly.plotly as py\n",
    "import plotly.graph_objs as go\n",
    "\n",
    "plotly.tools.set_credentials_file(username='abkds', api_key='KKuXHMUKu7EHg9kIZWrl')\n",
    "\n",
    "\n",
    "# Create random data with numpy\n",
    "import numpy as np\n",
    "\n",
    "N = 500\n",
    "xs = sampled_subreddits_embedded[:, 0]\n",
    "ys = sampled_subreddits_embedded[:, 1]\n",
    "\n",
    "# Create a trace\n",
    "trace = go.Scatter(\n",
    "    x = xs,\n",
    "    y = ys,\n",
    "    mode='markers+text',\n",
    "    text=sampled_subreddits\n",
    ")\n",
    "\n",
    "data_ = [trace]\n",
    "\n",
    "# Plot and embed in ipython notebook!\n",
    "py.iplot(data_, filename='basic-scatter')\n",
    "\n",
    "# or plot with: plot_url = py.plot(data, filename='basic-line')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Bayesian Personalized Ranking"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "from implicit.bpr import BayesianPersonalizedRanking\n",
    "\n",
    "params = {\"factors\": 63}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import logging\n",
    "import tqdm\n",
    "import time\n",
    "import codecs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BayesianPersonalizedRanking(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "model_name = 'bpr'\n",
    "output_filename = 'subreddits_recs_bpr'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:33<00:00,  2.98it/s, correct=97.91%, skipped=4.55%]\n"
     ]
    }
   ],
   "source": [
    "model.fit(comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "def bpr_related_subreddits(subreddit):\n",
    "    found = np.where(subreddits == subreddit)\n",
    "    if len(found[0]) == 0:\n",
    "        raise ValueError(\"Subreddit doesn't exist in the dataset.\")\n",
    "    _id = found[0][0]\n",
    "    return [(subreddits[i], v) for i, v in model.similar_items(_id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('dogs', 3.2827873),\n",
       " ('Dogtraining', 2.7844837),\n",
       " ('puppy101', 2.5864434),\n",
       " ('WiggleButts', 2.4180102),\n",
       " ('Paleo', 2.4142845),\n",
       " ('EatCheapAndHealthy', 2.3868768),\n",
       " ('germanshepherds', 2.3686364),\n",
       " ('AskVet', 2.3481765),\n",
       " ('Pets', 2.341671),\n",
       " ('TheGirlSurvivalGuide', 2.3099918)]"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "bpr_related_subreddits('dogs')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "users = data['user'].cat.categories.array.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "write_bpr_recommendations = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9a1b4f7e9a33434ea78449a7c34eea2c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=735834), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "user_comments = comments.T.tocsr()\n",
    "if write_bpr_recommendations:\n",
    "    # generate recommendations for each user and write out to a file\n",
    "    with tqdm.tqdm_notebook(total=len(users)) as progress:\n",
    "        with codecs.open(output_filename, \"w\", \"utf8\") as o:\n",
    "            for userid, username in enumerate(users):\n",
    "                for subredditid, score in model.recommend(userid, user_comments):\n",
    "                    o.write(\"%s\\t%s\\t%s\\n\" % (username, subreddits[subredditid], score))\n",
    "                progress.update(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Sample user recommendations\n",
    "\n",
    "We went through the user 'xkcd_transciber' list of subreddits, where he/she commented. Taking a view of the kind of subreddits followed by the user we see that the predictions are good. This is just one sample, we are saving the recommendations for all users in a file and will also write the AUC score function for getting the exact scores for the generated recommendations."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "def recommend_for_user(username):\n",
    "    sample_user_id = np.where(users == username)[0][0]\n",
    "    return [(subreddits[i], v) for i, v in model.recommend(sample_user_id, user_comments)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[('dwarffortress', 3.7335882),\n",
       " ('badphilosophy', 3.1564484),\n",
       " ('teslore', 3.1290364),\n",
       " ('Morrowind', 2.9986079),\n",
       " ('mountandblade', 2.9104137),\n",
       " ('Anarchism', 2.8900142),\n",
       " ('KerbalAcademy', 2.8693864),\n",
       " ('HPMOR', 2.8496687),\n",
       " ('shittykickstarters', 2.8411458),\n",
       " ('askphilosophy', 2.7882056)]"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "recommend_for_user('xkcd_transcriber')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def subreddits_interacted_by_user(username):\n",
    "    sample_user_id = np.where(users == username)[0][0]\n",
    "    _idlist =  comments.getcol(sample_user_id)\n",
    "    return [subreddits[idx] for idx, i in enumerate(_idlist.toarray()) if i != 0.0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Planetside',\n",
       " 'tifu',\n",
       " 'homestuck',\n",
       " 'Minecraft',\n",
       " 'worldbuilding',\n",
       " 'italy',\n",
       " 'AgainstGamerGate',\n",
       " 'firefly',\n",
       " 'tf2',\n",
       " 'MLPLounge',\n",
       " 'Futurology',\n",
       " 'sysadmin',\n",
       " 'GlobalOffensive',\n",
       " 'argentina',\n",
       " 'india',\n",
       " 'RandomActsOfGaming',\n",
       " 'books',\n",
       " 'worldnews',\n",
       " 'boardgames',\n",
       " 'CasualConversation']"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# sample 50 reddits with which xkcd_transcriber has interacted with.\n",
    "\n",
    "random.sample(subreddits_interacted_by_user('xkcd_transcriber'), 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c90c3113717242259a721ee69ba28cd9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "2f020d0befba451185505eb7808c9fe3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=735834), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "# set seed to get the same train and test set\n",
    "np.random.seed(42)\n",
    "\n",
    "filename = 'final_interactions_count'\n",
    "train_filename = 'interactions_5'\n",
    "\n",
    "def create_dataset():\n",
    "    data = defaultdict(lambda: [])\n",
    "    with open(filename) as csvfile:\n",
    "        datareader = csv.reader(csvfile, delimiter=' ')        \n",
    "        for subreddit, user, comments, _ in tqdm.tqdm_notebook(datareader):\n",
    "            data[user].append((subreddit, comments))\n",
    "    \n",
    "\n",
    "    f_train = open(train_filename, 'w')\n",
    "    \n",
    "    for user, items in tqdm.tqdm_notebook(data.items()):\n",
    "        np.random.shuffle(items)\n",
    "        if len(items) >= 2:\n",
    "            for item in items:\n",
    "                line = ' '.join(list(map(str, [item[0], user, item[1]]))) + '\\n'\n",
    "                f_train.write(line)\n",
    "    \n",
    "    f_train.close()\n",
    "        \n",
    "create_dataset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = []\n",
    "with open('interactions_5') as csvfile:\n",
    "    datareader = csv.reader(csvfile, delimiter=' ')\n",
    "    for subreddit, user, comments in datareader:\n",
    "        data.append([user, subreddit, 1])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1087036"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = pd.DataFrame.from_records(data)\n",
    "data.columns = ['user', 'subreddit', 'comments']\n",
    "\n",
    "data['user'] = data['user'].astype(\"category\")\n",
    "data['subreddit'] = data['subreddit'].astype(\"category\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# create a sparse matrix of all the artist/user/play triples\n",
    "comments = coo_matrix((data['comments'].astype(float), \n",
    "                   (data['subreddit'].cat.codes, \n",
    "                    data['user'].cat.codes)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<13500x285103 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 1087036 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "subreddits = data['subreddit'].cat.categories.array.to_numpy()\n",
    "users = data['user'].cat.categories.array.to_numpy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Number of users for BPR model: 285103\n",
      "Number of subreddits for BPR model: 13500\n"
     ]
    }
   ],
   "source": [
    "print('Number of users for BPR model: %s' % len(users))\n",
    "print('Number of subreddits for BPR model: %s' % len(subreddits))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the index and the reverse index for the users and subreddits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "subreddits_index = item_to_index(subreddits)\n",
    "users_index = item_to_index(users)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Extracting test set\n",
    "\n",
    "We will pluck out the test set, as per the strategy given in the paper [BPR: Bayesian Personalized Ranking from Implicit Feedback](https://arxiv.org/pdf/1205.2618.pdf), section 6.2 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_test_split(coo_comments):\n",
    "    \"\"\"\n",
    "    Omits random user subreddit interactions, zeros them out \n",
    "    and appends them to the test list.\n",
    "    \"\"\"\n",
    "    csr_comments = coo_comments.tocsr()\n",
    "    \n",
    "    data = defaultdict(lambda: [])\n",
    "    with open('interactions_5') as csvfile:\n",
    "        datareader = csv.reader(csvfile, delimiter=' ')        \n",
    "        for subreddit, user, comments in tqdm.tqdm_notebook(datareader):\n",
    "            data[user].append((subreddit, comments))\n",
    "    \n",
    "    train_set = []\n",
    "    test_set = []\n",
    "    \n",
    "    for user, items in tqdm.tqdm_notebook(data.items()):\n",
    "        np.random.shuffle(items)\n",
    "        test_item = items[0]\n",
    "        test_comments = items[1]\n",
    "        \n",
    "        test_subreddit = test_item[0]\n",
    "        # zero out a user item interaction\n",
    "        csr_comments[subreddits_index[test_subreddit], users_index[user]] = 0\n",
    "        \n",
    "        test_set.append([test_subreddit, user, int(comments)])\n",
    "        \n",
    "        for item in items[1:]:\n",
    "            train_set.append([item[0], user, int(item[1])])\n",
    "        \n",
    "    csr_comments.eliminate_zeros()\n",
    "    return train_set, test_set, csr_comments.tocoo()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ce1c954c01244483ad796996b0a73575",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=1, bar_style='info', max=1), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62de1ef131e741b7b99847896e07e97a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=285103), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train_set, test_set, comments = train_test_split(comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "801933"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_set)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "285103"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_set)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### AUC Metric\n",
    "\n",
    "We will implement the AUC Metric for evaluation of BPR based methods. We take the definition given in the paper [BPR: Bayesian Personalized Ranking from Implicit Feedback](https://arxiv.org/pdf/1205.2618.pdf), section 6.2 . AUC is defined as \n",
    " \n",
    "$$AUC = \\frac{1}{| U |} \\sum_u \\frac{1}{|E(u)|} \\sum_{(i, j) \\in E(u)} \\delta(\\hat{x}_{ui} - \\hat{x}_{uj}) $$\n",
    "\n",
    "where $$E(u) := \\{(i, j) | (u, i) \\in S_{test} ∧ (u, j) \\notin (S_{test} ∪ S_{train})\\}$$\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4f708fb827d74865b8860996d8c93387",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=801933), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b97ec8892ffd4f76b20f2d207b315a46",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=285103), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# create E(u) list for each user and store it use ids instead of names to store them\n",
    "E_u = defaultdict(lambda : set())\n",
    "\n",
    "for subreddit, user, _ in tqdm.tqdm_notebook(train_set):\n",
    "        E_u[users_index[user]].add(subreddits_index[subreddit])\n",
    "            \n",
    "for subreddit, user, _ in tqdm.tqdm_notebook(test_set):\n",
    "        E_u[users_index[user]].add(subreddits_index[subreddit])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train the bpr model \n",
    "from implicit.bpr import BayesianPersonalizedRanking\n",
    "\n",
    "params = {\"factors\": 63}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = BayesianPersonalizedRanking(**params)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<13500x285103 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 801933 stored elements in COOrdinate format>"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "comments"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:18<00:00,  5.82it/s, correct=96.78%, skipped=6.10%]\n"
     ]
    }
   ],
   "source": [
    "model.fit(comments)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "num_subreddits = len(subreddits)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "from utils import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auc(test_set, user_factors, subreddit_factors, subreddits, users):\n",
    "    \"\"\"\n",
    "    Returns the auc score on a test data set\n",
    "    \"\"\"\n",
    "    num_users = len(test_set)\n",
    "    total = 0\n",
    "\n",
    "    # treat the signal as 1 as per the implicit bpr paper\n",
    "    for subreddit, user, signal in tqdm.tqdm_notebook(test_set):\n",
    "        u = users_index[user]\n",
    "        i = subreddits_index[subreddit]\n",
    "\n",
    "        x_ui = user_factors[u].dot(subreddit_factors[i])\n",
    "\n",
    "        js = []\n",
    "\n",
    "        for j in range(0, num_subreddits):\n",
    "            if j != i and j not in E_u[u]:\n",
    "                js.append(j)\n",
    "\n",
    "        total += np.sum(np.heaviside(x_ui - \\\n",
    "                        user_factors[u].dot(subreddit_factors[js].T), 0)) / len(js)\n",
    "\n",
    "    return total / num_users"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4d4bf4cd348a45d4bf37eb285da8e7ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.7154472624571377"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "auc(test_set[:10000], model.user_factors, model.item_factors, subreddits, users)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_aucs_vs_factors():\n",
    "    factors = [8, 16, 32, 64, 128]\n",
    "    params_list = [{\"factors\": factor} for factor in factors]\n",
    "    \n",
    "    aucs = []\n",
    "    \n",
    "    for params in params_list:\n",
    "        model = BayesianPersonalizedRanking(**params)\n",
    "        model.fit(comments)\n",
    "        auc_ = auc(test_set[:10000], model.user_factors, model.item_factors, subreddits, users)\n",
    "        print(auc_)\n",
    "        aucs.append(auc_)\n",
    "    \n",
    "    return aucs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:16<00:00,  6.49it/s, correct=93.07%, skipped=6.06%]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "10321dc325c14558b9bb47ce3b3377ca",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\r",
      "  0%|          | 0/100 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.719584496581462\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:14<00:00,  7.39it/s, correct=95.73%, skipped=6.08%]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f15673cb8b1543e0805de7dd67da5f05",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7027533337412357\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:15<00:00,  6.66it/s, correct=96.84%, skipped=6.02%]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8090934de0814d1b8df30014b1aba2ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7139457583443933\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:23<00:00,  5.04it/s, correct=96.82%, skipped=6.04%]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bcd9790b37a740689e4b88b89a4d0f03",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7035268376229404\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 100/100 [00:22<00:00,  4.60it/s, correct=96.55%, skipped=6.11%]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f51b8a6481f4a478611ab4315bdb986",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7166421893585617\n"
     ]
    }
   ],
   "source": [
    "aucs_vs_factors = get_aucs_vs_factors()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.7127681415290543,\n",
       " 0.7169677827047857,\n",
       " 0.7230466831483704,\n",
       " 0.7053167273837033,\n",
       " 0.7169114986739286]"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aucs_vs_factors"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def auc(test_set, user_factors, subreddit_factors, subreddits, users):\n",
    "    \"\"\"\n",
    "    Returns the auc score on a test data set\n",
    "    \"\"\"\n",
    "    num_users = len(test_set)\n",
    "    total = 0\n",
    "\n",
    "    # treat the signal as 1 as per the implicit bpr paper\n",
    "    for subreddit, user, signal in tqdm.tqdm_notebook(test_set):\n",
    "        u = users_index[user]\n",
    "        i = subreddits_index[subreddit]\n",
    "\n",
    "        x_ui = user_factors[u].dot(subreddit_factors[i])\n",
    "\n",
    "        js = []\n",
    "\n",
    "        for j in range(0, num_subreddits):\n",
    "            if j != i and j not in E_u[u]:\n",
    "                js.append(j)\n",
    "\n",
    "        total += np.sum(np.heaviside(x_ui - \\\n",
    "                        user_factors[u].dot(subreddit_factors[js].T), 0)) / len(js)\n",
    "\n",
    "    return total / num_users\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_aucs_vs_factors_als():\n",
    "    factors = [8, 16, 32, 64, 128]\n",
    "    \n",
    "    aucs = []\n",
    "    \n",
    "    for factor in factors:\n",
    "        subreddit_factors, user_factors = alternating_least_squares(bm25_weight(comments), factor)\n",
    "        auc_ = auc(test_set[:10000], user_factors, subreddit_factors, subreddits, users)\n",
    "        print(auc_)\n",
    "        aucs.append(auc_)\n",
    "    \n",
    "    return aucs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:implicit:This method is deprecated. Please use the AlternatingLeastSquares class instead\n",
      "100%|██████████| 15.0/15 [00:04<00:00,  3.42it/s]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0eba320176bc47979d9ae18daed58015",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:implicit:This method is deprecated. Please use the AlternatingLeastSquares class instead\n",
      "  0%|          | 0/15 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7532502962975772\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15.0/15 [00:05<00:00,  2.82it/s]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "3c8e91cf11e14dbb8e1b87eb2c6a525b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:implicit:This method is deprecated. Please use the AlternatingLeastSquares class instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7646220234906469\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15.0/15 [00:06<00:00,  2.53it/s]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ab34abf4fe664a3bb6665f4f26775097",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:implicit:This method is deprecated. Please use the AlternatingLeastSquares class instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7811194828486229\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15.0/15 [00:09<00:00,  1.83it/s]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "88a4ffac893f48f4bd934ef18e3b4732",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:implicit:This method is deprecated. Please use the AlternatingLeastSquares class instead\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.7996316000860498\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 15.0/15 [00:16<00:00,  1.00s/it]\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fd86be3cbe3c447689c132393a3123ed",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=10000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "0.8152623487533477\n"
     ]
    }
   ],
   "source": [
    "aucs_als = get_aucs_vs_factors_als()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[0.7532502962975772,\n",
       " 0.7646220234906469,\n",
       " 0.7811194828486229,\n",
       " 0.7996316000860498,\n",
       " 0.8152623487533477]"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "aucs_als"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "aucs_tbpr = [0.9005884623283035,\n",
    " 0.9000449631079535,\n",
    " 0.8960811226825817,\n",
    " 0.8951720529233058,\n",
    " 0.89860491001835]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 58,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEWCAYAAACXGLsWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMi4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvhp/UCwAAIABJREFUeJzt3Xl8VOW9+PHPN3sC2UgCKAmBIFpcg/LTWpfCVVErdbltrUvr0lbqVevFtrfVXzeFq9LW3mp/tbVo3Voqta5ovS5VUnFrwYKACMoWEhIgBAgJIet8f3+cM8OZySQzk8xk4/vmNS/Oec5znvPMOZPzPc9zNlFVjDHGmJ4kDXQFjDHGDH4WLIwxxkRkwcIYY0xEFiyMMcZEZMHCGGNMRBYsjDHGRGTBwgxrInKaiHwiIk0icvFA1yeUiFSIyDe6mTZBRFREUtzx/xWRq/u3hsY4LFgME+5OZ4+IpIdJ/0ZI2nQRqfaMi4jcLCJrRGS/iFSLyF9E5Lj+qn8CzQV+raojVfW5vhYmIo+KSJsbfHaLyGsi8qk41DMiVT1fVR9z63GNiLwVRV3/uz/qlkgiskVEDohIo4jsFZF3ROR6EYlq/xUadE3vWLAYBkRkAnAGoMCFvSjiPuA/gZuBUcCRwHPABfGpYWxEJDmOxZUCH/ayHt3tXH6mqiOBccA24Pe9rNshpw877M+rajbO9pwPfB9b7/3KgsXwcBXwHvAoEFM3hYhMBm4ELlfVN1S1VVWbVXWhqs7vZp5rRGSTe6S3WUSu9Ey7TkQ+cqetFZET3fQpbitnr4h8KCIXeuZ5VER+KyIvich+YIaIpIvIPSKyVUR2iMgDIpLp5i8UkRfdsnaLyNJwR5kishEoA15wWwLpInK4iCx259sgItd58t8uIk+JyB9FZB9wTU/rTlUPAE8C5SHL/Zq7DvaIyCsiUuqZdo6IrBORBhH5NSCeacnud94lIpsICdb+VqKITAEeAE51v9fenurpmf8vIrLdXfabInKMZ9qjInK/iPzV3Xb/EJFJ7rQuR+beFquITBKRN0Sk3q37QhHJ8+TdIiLfF5FVwH4R+S8ReTqkbv9PRO6N9B1UtUFVFwNfBq4WkWPd+S8QkRUisk9EqkTkds9sb7r/73XX16mR6mzCUFX7DPEPsAG4ATgJaAfGeKZVAN8IyT8dqHaHrwcqY1jWCGAfcJQ7fhhwjDv8JZwj7f+DsxM8AudIMNWt4/8F0oB/Axo9ZTwKNACn4RzAZAD3AotxWjrZwAvA3W7+u3F2lqnu5wxAuqnvFuBsz/jfgd+4yygH6oCz3Gm3u+vvYrcemWHKexT4b8+6+APwgWf6xe53nQKkAD8E3nGnFbrr7otuvW8BOvzbx90W64AS93svwWktpoRuS5xA9laEbRWoqzv+NXddprvrd2VI3t3AyW69FwKL3GkTvPUIU5cjgHPccotwds73hmyDle73ysT5zewH8tzpKcBO4KRotqEnfSvwH57f9HHudjse2AFc3EP9e6yzfbp+rGUxxInI6Tg75CdV9X1gI3BFDEUUALUxLtYHHCsimapaq6r+bp5v4HTRLFPHBlWtBD4NjATmq2qbqr4BvAhc7inzeVV9W1V9QCtwHXCLqu5W1UbgLuAyN287zg6nVFXbVXWpunuAnohICXA68H1VbVHVlcBDwFc92d5V1edU1adOyyGc77pH841ued75v4kT1D5S1Q633uVu6+JzwFpVfUpV23F22Ns9816Ks8OqUtXdOEExblT1YVVtVNVWnMB4gojkerI8o6r/dOu9kJAWUw/lblDV19RpldYB/wN8NiTbr9zvdUBVa3F2zl9yp50H7HJ/v7GowQmqqGqFqq52t9sq4IkwdYi1zsbDgsXQdzXwqqrucsf/RHBXVAfOUaxXKs4OF6AeZ8cbFVXdj9MFcD1Q63Zb+E/wluAEq1CHA1VuIPCrxOnz96vyDBcBWcD7blfTXuBlNx3g5zhH76+63WG3Rln9wwF/8ImmHt25R1XzcI5YDwBHeaaVAvd56r0bp5U1zl1+oHw3wHmXd3jIeGUUdYmK28U1X0Q2ul1sW9xJhZ5s3sDVjBPgoyl7tIgsEpFtbtl/DCkXuq7Xx4CvuMNfwWmhxWoczvpFRE4RkSUiUiciDTi/z9A6xFpn42HBYghz+/AvBT7r9kVvx+naOEFETnCzbcXZqXlN5OCO6HWgWESmRbtcVX1FVc/BCTLrgAfdSVXApDCz1AAlIecVxuN0WQWK9QzvwtkJH6Oqee4nV52TyrhHx99R1TLg88C3ReSsKKpeA4wSkewo69EjVd2Kc2HAff7zKTjr4JueeuepaqaqvoPTgivxzy8i4h0Pne7WrdvFR1tP1xXARcDZQC4HfxPS3Qwe+93/szxpYz3Dd7v1OV5Vc3B2/qHlhtb3OeB495zDLJyWTNRE5P/gBAv/FWF/wum2LFHVXJxuSn8dwq2raOpsPCxYDG0XA53A0ThdBuU4feVLcU56A/wZuFZEThbHkTgBZRGAqn6C04f/hDiX1KaJSIaIXBbuiF1ExojIhSIyAqe7qMmtAzhdOt8VkZPcZR3hdr/8A2eH8z0RSRWR6Tg7+UXhvpTbAnkQ+KWIjHaXO05EznWHZ7llC845gE5PHbqlqlXAO8Dd7nc8Hvg6Me6oQsp8DScIzXaTHgBu8588FpFcEfF3t/wVOEZE/t09WXwzwTvdJ4GbRaRYRPKBnlpMO3CCfFqUVc3G2V71ODv9u6KcD7ebZhvwFbeF8jWCDwqycX4He0VkHPBfUZTZAjyFs5P/pxt4IxKRHBGZhfPb+aOqrvbUYbeqtojIyQR3xdbhdJ2W9aXOhzoLFkPb1cAjqrpVVbf7P8CvgStFJEVVX8HZ6TyCcxL5JZwugAWecm5257kf2IvTlXQJzknlUEnAd3B2kLtx+nlvAFDVvwB34uwAGnGOHkepahvOJb3n47QafgNcparrevhu38fpanrP7Sb4Gwe7eya7403Au8BvVLUi4tpyXI5zVF0DPAv8xN3h98XPcQJhuqo+C/wUWOTWew3O98btKvwSzqWf9e73eNtTzoPAK8AHwL+AZ3pY5hs4lwRvF5FdPeTzH1U/jtOa3Aasxbl6LhbX4exQ64FjcIKu3x3AiTi/r79GqLfXYzgnpaPpgnpBRBpxWm4/wDnHcK1n+g3AXDfPj3ECLwCq2ozzu3zb7R78dB/qfMiSKM4LGmOGIBF5BnhTVSNekjoQRGQ8TjfmWFXdN9D1MT2zloUxw5DbtXI6sHyg6xKOe/7q2ziX51qgGAISFixE5GER2Skia7qZLiLyK3FujFol7s1b7rSrxXmezydiz8IxJiYicgOwAqeLssdHggwE93zXPpz7HH4ywNUxUUpYN5SInInTp/y4qh4bZvrngG/hXHt+CnCfqp4iIqNwjoam4fS3vo9zs86ehFTUGGNMRAlrWajqm7jXQHfjIpxAoqr6HpAnIocB5wKvuTdj7QFew7lpxxhjzAAZyKcwjiP4Rp1qN6279C5EZDbuJYuZmZknlZSUhMs2KPh8PpKS7BRRtGx9DT62TWIzVNbXxx9/vEtViyLlG8hgEe4GGO0hvWui6gLcS0CnTZumy5cPynN5AFRUVDB9+vSBrsaQYetr8LFtEpuhsr5EJKonBQxk2Ksm+G7VYpxr37tLN8YYM0AGMlgsBq5yr4r6NNDgPmDsFWCmiOS7d7HOdNOMMcYMkIR1Q4nIEziPDS4U561sP8F9oJ2qPoBzJ/HncO7Sbca9G1NVd4vIPGCZW9Rc9wmcxhhjBkjCgoWqXh5huuK8dCfctIeBhxNRL2OM8Wpvb6e6upqWlpa4lpubm8tHH30U1zL7IiMjg+LiYlJTQx9CHR17J60x5pBWXV1NdnY2EyZMwHk2ZXw0NjaSnZ0dOWM/UFXq6+uprq5m4sSJvSpj8F/XZYwxCdTS0kJBQUFcA8VgIyIUFBT0qfVkwcIYc8gbzoHCr6/f0YKFMcaYiCxYGGNMjJZsXsKEeyewZPOSuJSXnJxMeXk5J5xwAieeeCLvvOO8LmTLli1kZmZSXl7O0UcfzfXXX4/P5+uSftVVV9He3h5hKX1jwcIYY2KwZPMSZj0xi8qGSmY9MSsuASMzM5OVK1fywQcfcPfdd3PbbbcFpk2aNImVK1eyatUq1q5dy3PPPReUvnr1aqqrq3nyySe7Kz4uLFgYY0yU/IGiub0ZgOb25rgFDL99+/aRn5/fJT0lJYXPfOYzbNiwISg9OTmZk08+mW3btnWZJ57s0lljjHHNeXkOK7evDDttT8se1uxcg099QenN7c2c/YezOXb0seRnHNzJd3Z2Ot1LY8u597yeX1Z44MABysvLaWlpoba2ljfeeKNLnubmZl5//XXmzp0blN7S0sI//vEP7rvvvmi/Zq9Yy8IYY6Kwftf6LoHCz6c+1u9a3+uy/d1Q69at4+WXX+aqq67C/66hjRs3Ul5ezmmnncYFF1zA+eefH5ReUFDA+PHjOf7443u9/GhYy8IYY1w9tQBCu6C8slKzePHyF5kxcUYgrbc35Z166qns2rWLuro64OC5iVD+9NraWqZPn87ixYu58MILY15etKxlYYwxUZgxcQYvXv4iWalZQenhAkVfrFu3js7OTgoKCqLKf9hhhzF//nzuvvvuuCy/OxYsjDEmSqEBI16Bwn/Oory8nC9/+cs89thjJCcnRz3/xRdfTHNzM0uXLu1TPXpi3VDGGBMDf8C49vlreeSiR+LSoujs7AybPmHCBNasWRMxXUT44IMP+lyPnliwMMaYGM2YOIMtc7YMdDX6lXVDGWOMiciChTHGmIgsWBhjjInIgoUxxpiIEhosROQ8EVkvIhtE5NYw00tF5HURWSUiFSJS7JnWKSIr3c/iRNbTGGNMzxIWLEQkGbgfOB84GrhcRI4OyXYP8LiqHg/MBbx3lRxQ1XL3k7jbEo0xZhB49tlnERHWrVsHOI8nP/bYY7vke++99zjllFMoLy9nypQp3H777f1Sv0S2LE4GNqjqJlVtAxYBF4XkORp43R1eEma6McYMOrWNtXz20c+yvWl73Mp84oknOP3001m0aFGP+a6++moWLFjAypUrWbNmDZdeemnc6tCTRAaLcUCVZ7zaTfP6APiCO3wJkC0i/nvcM0RkuYi8JyIXJ7CexhgTk3lvzuOtrW8x7+/z4lJeU1MTb7/9Nr///e8jBoudO3dy2GGHAc7jyY8+OrTDJjESeVNeuBe+asj4d4Ffi8g1wJvANqDDnTZeVWtEpAx4Q0RWq+rGoAWIzAZmA4wZM4aKioo4Vj++mpqaBnX9BhtbX4PPcN0mubm5NDY2AvD9Jd9ndd3qHvO3dbaxvHY5Pnw8sPwBlm9bTlpyWpd8qoqIcFzRcfx0xk97LHPRokWcddZZHHbYYeTm5rJ06VLy8/Px+XyBuvndcMMNHHXUUZx++umcffbZXHHFFWRkZET1XVtaWnq9DRMZLKqBEs94MVDjzaCqNcC/A4jISOALqtrgmYaqbhKRCmAqsDFk/gXAAoBp06bp9OnTE/E94qKiooLBXL/BxtbX4DNct8lHH30UeDpsWlpaxGcyVe2tQt3jXkWpbqxmcsHkLvn877NIS0uL+PTZ5557jjlz5pCdnc2VV17J4sWLufHGG0lKSuoy75133snXvvY1Xn31VRYtWsSzzz4bdQDIyMhg6tSpUeUNlchgsQyYLCITcVoMlwFXeDOISCGwW1V9wG3Aw256PtCsqq1untOAnyWwrsYYE/ElRbWNtZT9qiwoWOxp2cOiLy5i7MixQXmjfUR5fX09b7zxBmvWrEFE6OzsRES44YYbup1n0qRJ/Md//AfXXXcdRUVF1NfXR/2U2t5K2DkLVe0AbgJeAT4CnlTVD0Vkroj4r26aDqwXkY+BMcCdbvoUYLmIfIBz4nu+qq5NVF2NMSYa896c1+UFSJ3a2adzF0899RRXXXUVlZWVbNmyhaqqKiZOnEh1dXXY/H/9618DL0b65JNPSE5OJi8vr9fLj1ZCHySoqi8BL4Wk/dgz/BTwVJj53gGOS2TdjDEmVu9Wv0tbZ1tQWltnG+9Uv9PrMp944gluvTX4NrQvfOEL3HXXXaxfv57i4sDtZ/zyl7/k6aef5pZbbiErK4uUlBQWLlwY0+PMe8ueOmuMMVFa8c0VcS8z3PmGm2++mZtvvjls/i996Utxr0M07HEfxhhjIrJgYYwxJiILFsYYYyKyYGGMMSYiCxbGGGMismBhjDEmIgsWxhgzgPbu3ctvfvMbwHkseWZmJuXl5Zxwwgl85jOfYf369YBziW1ubi5Tp05lypQp3HHHHV3SP/WpT/Hd7343IfW0YGGMMTFYuHohE+6dQNIdSUy4dwILVy/sU3neYAHOozxWrlzJBx98wNVXX81dd90VmHbGGWewYsUKli9fzh//+Efef//9oPQVK1bw4osv8vbbb/epTuHYTXnGGBOlhasXMvuF2TS3NwNQ2VDJ7BdmA3DlcVf2qsxbb72VjRs3Ul5ezuTJwQ8k3LdvH/n5+V3mGTFiBCeddBIbN25k9OjRgXR/q2Tbtm29qktPLFgYY4xrzstzWLl9ZbfT36t+j9bO1qC05vZmvv7813nw/QeD0v1PnS0fW97jAwrnz5/PmjVrWLlyJVu2bGHKlCmUl5fT2NhIc3Mz//jHP7rMU19fz3vvvcePfvQj6urqAul79uzhk08+4cwzz4z2K0fNuqGMMSZKoYEiUnpv+LuhNm7cyL333svs2bMD05YuXcrUqVOZOXMmt956K8ccc0wg/fjjj2fs2LHMmjWLsWPHdld8r1nLwhhjXJEeUT7h3glUNlR2SS/NLaXimoqgtGgfUd6TCy+8kGuvvTYwfsYZZ/Diiy92yedP//jjjzn99NO55JJLKC8v79OyQ1nLwhhjonTnWXeSlZoVlJaVmsWdZ93ZzRyRZWdnd3kbnt9bb73FpEmToi7ryCOP5LbbbuOnP+35zXy9YS0LY4yJkv8k9g9e/wFbG7YyPnc8d551Z69PbgMUFBRw2mmnceyxxzJlypTAyW5VJS0tjYceeiim8q6//nruueceNm/ezMSJE3tdr1AWLIwxJgZXHndln4JDOH/6058i5pk+fXrY19qGpmdmZibkaijrhjLGGBORBQtjjDERHfLBIt53Yxpjhh7/O62Hs75+x4QGCxE5T0TWi8gGEbk1zPRSEXldRFaJSIWIFHumXS0in7ifqxNRP//dmJUNlSgauBvTAkb3LLia4SYjI4P6+vohGTDqm+tZtWMVy2uWs2rHKuqb68PmU1Xq6+vJyMjo9bISdoJbRJKB+4FzgGpgmYgsVtW1nmz3AI+r6mMi8m/A3cBXRWQU8BNgGqDA++68e+JZxx+8/oPAbft+ze3NzHl5DmlJaYgIgvi/D4L0+v8P9nyAbtY+lzOQ/z+19inmvDyHAx0HgPg86sCYgVZcXEx1dXXQndDx0NLS0qedcyT72/ZTfyA4yNVKLQWZBYxIG9Elf0ZGBsXFxV3So5XIq6FOBjao6iYAEVkEXAR4g8XRwC3u8BLgOXf4XOA1Vd3tzvsacB7wRDwruLVha9j0Xc27uPSpS+O5KMeq+Bc50Jrbm/nqM1/l5v+9mYyUDNKT00lPSSc9Od0Zd4fTU9IPTg8dDxlOT05n045N7Fq7q9vp4cpOSRp8F/ctXL0wrpdZDncDsb5SU1PjeompX0VFBVOnTu3VvKpKp3bS1tlGa0crbZ1tgU9rpzP++YWfZ3vT9i7zluaWsmXOlj7WvqtE/nWNA6o849XAKSF5PgC+ANwHXAJki0hBN/OOC12AiMwGZgOMGTOGioqKmCo4On00O1p3dEkvSCvg58f/PBCx1f0HzkbszXjzgWYyMzN7PX9gHA2k9ff4/RvvD7seFeXM/DNp87XR7munTdto72inra2NBl8D7b522rX94HRfW9B4h3aE30Dretp6XSWRRGpSKqlJqaQlpZEq7v+e8aDpSamkSVrQuH+ewLgnLVy5/rTQ8lMlldd3vs49H99Dq895FERlQyVff+7rfLT2I84ec3ZsX24QaGpqivlvLBZ/2/G3fltfndpJh6+Ddm0P/Ab9/7f52nqe5hlv9wXnadf2wLzNbc3M+2geHb6OiOWGztvuaw/8DcZqa8PWhGynRAYLCZMW+u2/C/xaRK4B3gS2AR1RzouqLgAWAEybNk3DXYPck18U/CLoCZLg3I1536z74n40U1FREfYa6aHkxXtf7PZRB89e92yvy/Wpj7bONlo6WmjtaKW1s5U3336TE046gdbO1kCad7p3uLXDHQ/N2828rR2tNHU0OeNhpidSq6+V+evns3D7QpIkCUFIkiRnWDzDMaTHo4wu6WHy1NbWUjyuOGHL/82W3wQChXd9/Xrzr9FCDRxRhx5hB8Y9R+A9TWvrbKNTO+O+bVOTUklLTiM9JZ205DS0XckekU16cjppKWmkJaeRlZJFWnJa4JOenN7zeEp6t9O++eI3qWvu2nU2Pnd8QvY1iQwW1UCJZ7wYqPFmUNUa4N8BRGQk8AVVbRCRamB6yLwV8a5gIu7GHM7uPOvOsMG1L486AEiSJDJSMshIOdi/W5xVzHFjjutTub2hqrT72qMLRhGmz3tzXthldGonZ5aeiU99+NSHqh4cRnuV3u5rD6T3toxI6a1trSTvTg6bv7u0eGhobeBn7/ws7E7TuzNNS04jIyWDnPSc4OlJ4fPGsmPuad70lHRSk1IRCT7GTfQBYnNHc0L+HruTyGCxDJgsIhNxWgyXAVd4M4hIIbBbVX3AbcDD7qRXgLtExP8g95nu9LhLxN2Yw9WhEFxFJLATIL1vZT3+wePdtsQeu/ixvhU+AHqz84slKJ3wwAlU76vuUsb43PFUzum6Hg91/f33mLBgoaodInITzo4/GXhYVT8UkbnAclVdjNN6uFtEFKcb6kZ33t0iMg8n4ADM9Z/sNgPLgmv0EtUSG0pEhGRJJpnkiHnnnz0/7Pq666y7epjr0Naff48JvXxEVV8CXgpJ+7Fn+CngqW7mfZiDLQ1jhpxDoSUWT7a+BrfBd62hMcOItcRiY+tr8DrkH/dhjDEmMgsWxhhjIrJgYYwxJiILFsYYYyKyYGGMMSYiCxbGGGMismBhjDEmIgsWxhhjIrJgYYwxJiILFsYYYyKyYGGMMSYiCxbGGGMismBhjDEmIgsWxhhjIrJgYYwxJiILFsYYYyKyYGGMMSaihAYLETlPRNaLyAYRuTXM9PEiskREVojIKhH5nJs+QUQOiMhK9/NAIutpjDGmZwl7raqIJAP3A+cA1cAyEVmsqms92X4IPKmqvxWRo3He1z3BnbZRVcsTVT9jjDHRS2TL4mRgg6puUtU2YBFwUUgeBXLc4VygJoH1McYY00sJa1kA44Aqz3g1cEpIntuBV0XkW8AI4GzPtIkisgLYB/xQVZeGLkBEZgOzAcaMGUNFRUXcKh9vTU1Ng7p+g42tr8HHtklshtv6SmSwkDBpGjJ+OfCoqv5CRE4F/iAixwK1wHhVrReRk4DnROQYVd0XVJjqAmABwLRp03T69Olx/xLxUlFRwWCu32Bj62vwsW0Sm+G2vhLZDVUNlHjGi+nazfR14EkAVX0XyAAKVbVVVevd9PeBjcCRCayrMcaYHiQyWCwDJovIRBFJAy4DFofk2QqcBSAiU3CCRZ2IFLknyBGRMmAysCmBdTXGGNODhHVDqWqHiNwEvAIkAw+r6ociMhdYrqqLge8AD4rILThdVNeoqorImcBcEekAOoHrVXV3oupqjDGmZ4k8Z4GqvoRzOaw37cee4bXAaWHmexp4OpF1M8YYEz27g9sYY0xEFiyMMcZEZMHCGGNMRBYsjDHGRGTBwhhjTEQWLIwxxkRkwcIYY0xEFiyMMcZEZMHCGGNMRBYsjDHGRGTBwhhjhrDaxlo+++hn2d60PaHLsWBhjDFD2Lw35/HW1reY9/d5CV1OQh8kaIwxpm9aO1qp3ldN9b5qqvZVOf83VFHdWM2m3ZtYU7cGgEdWPsKPPvsjxo4cm5B6dBssRORcIFtVnwpJvxLYqaqvJaRGxhhziGjtaKWmsSYoCASG3f937t/ZZb78jHyKc4rZ17qPJEnCpz46tZN5f5/H/Rfcn5C69tSyuAP4fJj014FnAQsWxhjTjXZfO1v2bnFaAd5WgScw7Ni/o8t8uem5lOSWUJJTwkmHnURJTgnFOcWU5Dr/F+cUMzJtJLWNtZT9qgyf+gBo62xLaOuip2CRpap1oYmqul1ERsS9JsYYM0S0d7ZT01jTZefvDQg7mnagSzVovpz0nMDOv3xMeSAI+NOKc4rJTs+Oqg7z3pwXCBR+iWxd9BQsMkQkRVU7vIkikgpkxr0mxhgzCHT4OqhtrO2xa6i2sRYlOBBkp2UHjv6PH3M8Hbs7OOP4M4JaBTnpOXGr57vV79LW2RaU1tbZxjvV78RtGV49BYtncF55epOq7gdwWxS/cqcZY8yQ0unrpLaptseuodqm2i5H7CNSRwRaAMdMOoaSnJJAAPC3CnIzcoPmqaioYPqJ0xP2XVZ8c0XCyg6np2DxQ+C/gUoRqQQEKAF+D/womsJF5DzgPpx3cD+kqvNDpo8HHgPy3Dy3uq9iRURuA76O8w7um1X1lRi+lzFmiKptrOWypy/jz1/8c0x9752+TrY3be+xa6i2sZZO7QyaLys1K7DzP2fSOQfPEXgCQm56LiIS7686pHQbLNzup1tF5A7gCDd5g6oeiKZgEUkG7gfOAaqBZSKy2H3vtt8PgSdV9bcicjTO+7onuMOXAccAhwN/E5EjVUO2sjFm2PHeN+Dve/epjx1NO6jaVxW2VVDVUEVNY02XQJCZkhnY4Z818awuQaAkp4S8jLxDPhBEo6dLZ/89JEmBPBFZqaqNUZR9Mk5w2eSWtwi4CPAGCwX8nXi5QI07fBGwSFVbgc0issEt790olmuMGWJaOlrYsncLy2uW89C/HsKnPn73/u9YVrOMnft3sq1xGx2+oNOnZKRkBHb4MybOoDi7OCgIlOSWkJ+Rb4EgTnrqhgp32ewo4HgR+bqqvhGh7HFAlWe8GjglJM/twKsi8i1gBHDr7tLIAAAcAElEQVS2Z973QuYdF7oAEZkNzAYYM2YMFRUVEao0cJqamgZ1/QYbW1+DT1+2iaqyu203NS011LbUUnuglpqWGra3bKfmQA272nZ1madTO9m0axMn5p3IaTmnMTpjNEVpRc7/6UXkpOR0DQRNzmdPzR72sKdXdY2X4fYb7qkb6tpw6SJSCjxJ1x1/l6zhig0Zvxx4VFV/ISKnAn8QkWOjnBdVXQAsAJg2bZpOnz49QpUGTkVFBYO5foONra/BJ9I22d+2n817N7Npz6agz+a9m9m8ZzMHOg72YAvCuJxxlOWXceLEEynLKyM/M5/vvPqdoCt8mn3NPP7VxxN2V3IiDbffcMyP+1DVSvfy2UiqcU6I+xVzsJvJ7+vAeW6574pIBlAY5bzGmH7UqZ1UNVQFB4O9B4dD7zTOTsumLL+MowqO4vwjzqcsvyzwKc0tJT0lPSj/DX+9IewyE3lXsolezMFCRD4FtEaRdRkwWUQmAttwTlhfEZJnK3AW8KiITAEygDpgMfAnEfkfnBPck4F/xlpXY0xsGloawrYONu3ZxOY9m+l48+B5g2RJZnzueCbmT+TCIy8MCgZl+WWMyhwV0/mC/r5vwMSmpxPcL9C162cUcBjwlUgFq2qHiNwEvIJzWezDqvqhiMwFlqvqYuA7OPdy3OIu6xpVVeBDEXkS52R4B3CjXQllTN+1d7ZTta+KzXs2h20d7D6wOyj/qMxRlOWXUT62nGkjpjH9hOmBYFCSU0JqcjSdDNHp7/sGTGx6alncEzKuwG6cgPEVorgyyb1n4qWQtB97htcCp3Uz753AnZGWYYw5SFXZfWB30PkCb+tga8PWoMtLU5NSmZA3gbL8MqYdNi2oZTAxfyJ5GXmBvBUVFUw/afoAfCszGPR0gvvv/mERKcfpQroU2Aw8nfiqGWPCae1opbKhMqh7yNs62Ne6Lyj/6BGjKcsv49SSU7niuCuCAsK47HEkJyUP0DcxQ0lP3VBH4pxnuByoB/4MiKrO6Ke6GTMsxHpHsqqyc//OLlcU+Yer91UHPZcoIyWDiXkTKcsv44zxZwSG/a2DkWkjE/n1zCGip26odcBS4POqugHAPbdgjIlBuDuSm9ub2bJ3S7cBobm9OaiMw7MPpyy/jBkTZ1CWdzAQlOWXMXbkWJLEXnppEqunYPEFnJbFEhF5GVhE+PsfjDEh/G83+1ftv7rckVy9r5raptqg/CNSR1CWX8ak/EmcU3bOwZZB3kQm5E0gM9Ue9GwGVk/nLJ4FnnWfNHsxcAswRkR+Czyrqq/2Ux2NGVRUlfoD9Wxt2Br4VO6tZOu+g+Pbm7Z3ma9TO9m2b1vQPQf+1kFRVpE9lsIMahHvs3AfT74QWCgio4AvAbcCFizMsNTW2Ub1vmonAHgCwtZ9WwNp3ruRwXlg3fjc8YzPHc8Fky+gNLeUnPQcvve37wXdO7CnZQ93nnXnkLwj2RzaYropT1V3A79zP8YMOf5LS4NaBQ3BQWF70/YuL7YZM2IMpXmlHDfmOC6YfEEgMIzPHU9pXikFmQVdWgZ2R7IZTmK+g9uYwayts41t+7Z1CQDez/72/UHzZKRkBHb85x9xPqV5pUHBoDinmIyUjJjrYnckm+HEgoUZMlSVPS17uuz8vYEh3OsuR48YTWluKUcXHc15R5wX3CrILaUwqzAh5wvsjmQznFiwMINGe2c72xq3sbVhK6/ueJW333w7cK7AHwya2pqC5klPTg/s+M+ddG4gAHhbBXYlkTF9Z8HCxKw3r71UVfa27O2xVVDTWBPcKlgHRVlFlOaV8qnCTzGzbGaXcwV2FZEx/cOChYlZuJvM2jvbqWmsCXvC2P9pbAt+wWJaclpgx3/OpHMYnzM+cL6gdl0tXzzni9YqMGaQsGBhetTe2c6u5l3UNddRt7+OT+o/CbrJ7J81/2R703ZqGmvwqS9o3sKsQsbnjufIgiM5u+zsoFbB+NzxjB4xuts7jyu2VligMGYQsWBxiNnftp+65jonAOyvCwSBQEBwx/159rbs7basTu2kZl+N0yoIOVdQkltCVmpWP34zY0wiWbAYwnzqo6GlocsOPhAEQgPB/rouN5P5pSalUphVSNGIIoqyiph2+DQKMw+OF40oIokkrnjmClo7D777ak/LHuafPd9uMjNmmLNgMYh0+DoCO3vvDj6w0z/QtTXQ2c07oUakjgjs6EePGM0xo49xdvrujr8wqzAwXJRVRE56TsQTxTf89YYul6XaTWbGHBosWNC7q3ui0dzeHNj5/3P3P6n6oCrs0b9/fE/Lnm7Lys/ID+zYjxh1BKcWn9plh+8NAono77ebzIw5dFmwIPzVPaFUlYbWhvD9/N4uIM946GOmWe38l5KUErSjLx9b3uNRf0FWASlJA7+p7CYzYw5dCd0Dich5wH047+B+SFXnh0z/JeB/mVIWMFpV89xpnQR2r2xV1QsTUccte7bw+xW/x6c+HvzXg2SmZtLa0Rr2ZG+HryNsGVmpWUE7+CmFUyjKKgo6B1C1vopzTz+XwqxC8jLy7N4AY8yQkrBgISLJwP3AOUA1sExEFrvv3QZAVW/x5P8WMNVTxAFVLU9U/fzu+Psdga6Vdl87v3j3F+Rl5AV2/GX5ZZw87uSgI31vECgaURTVVT8V2yuYXDA50V/HGGMSIpEti5OBDaq6CUBEFgEXAWu7yX858JME1qeL2sZaFn24KCgtMyWTj278yK7uMcYYj0QGi3FAlWe8GjglXEYRKQUmAm94kjNEZDnQAcxX1efCzDcbmA0wZswYKioqYqrgLz/+JR2dwV1L7Z3tXP/E9cyZPCemsiJpamqKuX6HMltfg49tk9gMt/WVyGARrlNew6SB8/rWp1SDrgMdr6o1IlIGvCEiq1V1Y1BhqguABQDTpk3T6dOnx1TBW9bfQocGB4sO7aDSV0msZUVSUVER9zKHM1tfg49tk9gMt/WVyGBRDZR4xouBmm7yXgbc6E1Q1Rr3/00iUoFzPmNj11l7z67uMcaY6IR/ME98LAMmi8hEEUnDCQiLQzOJyFFAPvCuJy1fRNLd4ULgNLo/12GMMSbBEtayUNUOEbkJeAXn0tmHVfVDEZkLLFdVf+C4HFikqt4uqinA70TEhxPQ5nuvojLGGNO/Enqfhaq+BLwUkvbjkPHbw8z3DnBcIutmjDEmeonshjLGGDNMWLAwxhgTkQULY4wxEVmwMMYYE5EFC2OMMRFZsDDGGBORBQtjjDERWbAAlmxewoR7J7Bk85KBrooxxgxKh3ywWLJ5CbOemEVlQyWznphlAcMYM6T018HuIR0s/IHC//rT5vZmCxjGmCGjPw92D9lgERoo/Jrbmzn3j+fyvde+x+ubXmfD7g20drQOUC3NcGDdnCYR+vtgN6HPhhrMrn3+2i6Bwq/d187P3/k5P3/n54G0sSPHUppbSmleKaW5pYzPHR80npuR219VN0OI9w961hOzePHyF5kxcUbkGY3pwSsbXuGSP1/CgY4DQemJ/J0dssHikYseCduyAMhKyeLBzz/I4TmHU7m3ksqGSrY2bKWyoZIVtSt4ft3ztHYGtzZy0nOCgkcgoLjjPvX111czg0R3R34WMEw4nb5O6prr2N60PeKnobWh23Ka25u59vlr2TJnS1zrd8gGixkTZ/Di5S92CRhZqVkR/5h96mPn/p1OAHGDSeXeSrbuc8bf2voWe1v2Bs2TKqmUrvG0SEJaKSW5JaQlpyXs+5r48qmPhpYG6prr2NW8i7r9zv+7mndR11zH6p2r+dumv3U5SGhub+acP5zDJZ+6hKmHTaUgs4CCrIIu/2ekZAzQNzPxpKo0tDZEFQDqmuvCHlRmp2UzduRYxo4cy/FjjmfmpJk0tzfzh1V/oK2zrUv+rNQsHrnokbh/Fwl+jcTQNW3aNF2+fHnM83mP/qIJFNHa17rPCSBui2Tp6qVojgZaKbWNtajnLbOCcFj2YWG7uErznLSc9Jw+12uo6O9XUrZ2tAZ2/N6dfyAYhAkKnUFvAT4oKzWL1o7WbqeDs72127cMO2WEDSTucGFWYZfpuem5iIR7m3F8DLfXhPbFgfYD7Ni/o8ed/5ZdW9jbsbdLLwRAalJqIAD09BkzYgwj0kaErUO486692YeJyPuqOi1SvkO2ZeHnb2Fc+/y1PHLRI3HrHshJz+G4Mcdx3BjntRxH7z866A+ttaOV6n3VB7u4/C2UhkqW1yzn2XXPdjlqyMvIOxg8csYHBZPS3FJGjxid0J3FUNHdUX93O/665jqa2prCliUIBVkFFGUVUZhVyFGFR3Fa5mkUjXDG/en+8cKsQrJSs7q9gAIO/kGfWnIq9c311B+o7/5/d7hqexX1zfXsPrC72yCTLMmMyhzVbYDx/l+YVUhBVgGjMkdZi9bV124gQSgaURTY0Z+QdwLlk8rDBoG8jLw+/62G9o7E82A3nEO+ZdFfYj0q86mPHU07DnZxuS0U/3hlQyX7WvcFzZOenB50niT0vElxTjGpyal9/i5LNi+Je3AN5V1f3qP+Ljv+/XXsOhCcXt9c3+NRf+gOPjAeZsefn5FPclJyr75DvI78vHzqY2/LXuqb69nVvKvHAOP9v6Wjpdsys9OyowowWz7awszTZ1KQVUB2WnbCDkzi+ftSVfa27A2/098f0g20vy5sIM5Jzwne2Y8I3wooGlFEStLB4+/+aon1dX1F27KwYNFPEvHDaWhpCAoegYDijm9v2h6UXxAOzz48KJj4u7j8wyPTRva4zL522/l3duF2/N7hTds30ZrcGtNRf9GIIgozwwSBkKP+/pSobs5YNbc3h229BAJOmAATet7NKzUplVGZowItlO4CjPf/UZmjgnam4US7vg60H+j+yD8kCITr109LTosqAIwZOabXv5mh0m03KIKFiJwH3IfzDu6HVHV+yPRfAv5fQhYwWlXz3GlXAz90p/23qj7W07IOxWARSUtHi9PVFeYkfGVDJVUNVbT72oPmGZU5qtuT8FX7qvjqs18NPlJOyeKRix9h8qjJXXb4fTnqT25J5siSI7s96i/KKiIvI6/XR/39qT9aYonQ4etgz4E9gQBS8c8KDp90eNhWjLeVE/qb8spNzw0KIIFzL5kF1B+o57fLfxu0c09NSmXWkbNITU4NCgChrWro2g3UUxCIRzdQJBYsoq9AMvAxcA5QDSwDLlfVtd3k/xYwVVW/JiKjgOXANECB94GTVHVPd8uzYBG7Tl8n25u2d2mReMcb2xp7VXaSJAX6xntz1D8Y19ehLpptoqo0tTVFdR7G+3+4nb9XcU4xZfllMXUDDbSh8hseDCe4TwY2qOomt0KLgIuAsMECuBz4iTt8LvCaqu52530NOA94IoH1PeQkJyUzLmcc43LGcWrJqV2m+/t7KxsqmfmHmdQ113VbVlFWEc98+Zkhd9Rv4ktEyE7PJjs9mwl5E6Ker/TeUrY2bO12erIk8/dr/h6HGpreSmSwGAdUecargVPCZRSRUmAi8EYP844LM99sYDbAmDFjqKio6HOlE6WpqWlQ1y+S2464jdvW3Earr+tlgOlJ6dx2xG10bOpgu/uvr4b6+hqOErlN5pTO6fH39Z+l/znkfg/D7TecyGARrkOwuz6vy4CnVAOd2VHNq6oLgAXgdEMN5ibfUGmSdmc60ykvL4/71T3dGerrazhK5Dbp799Xfxhuv+FEPkiwGijxjBcDNd3kvYzgLqZY5jX9xH9dt//cwlD+QzaDj/2+BrdEBotlwGQRmSgiaTgBYXFoJhE5CsgH3vUkvwLMFJF8EckHZrppZoD5/6BLc0vtD9nEnf2+Bq+EdUOpaoeI3ISzk08GHlbVD0VkLrBcVf2B43JgkXouy1LV3SIyDyfgAMz1n+w2A2/GxBlxf0iZMX72+xqcEnqdmaq+BLwUkvbjkPHbu5n3YeDhhFXOGGNM1A7Zlx8ZY4yJngULY4wxEVmwMMYYE5EFC2OMMRFZsDDGGBORBQtjjDERWbAwxhgTkQULY4wxEVmwMMYYE5EFC2OMMRFZsDDGGBORBQtjjDERWbAwxhgTkQULY4wxEVmwMMYYE5EFC2OMMRFZsDDGGBORBQtjjDERJTRYiMh5IrJeRDaIyK3d5LlURNaKyIci8idPeqeIrHQ/i8PNa4wxpn8k7B3cIpIM3A+cA1QDy0Rksaqu9eSZDNwGnKaqe0RktKeIA6panqj6GWOMiV4iWxYnAxtUdZOqtgGLgItC8lwH3K+qewBUdWcC62OMMaaXEhksxgFVnvFqN83rSOBIEXlbRN4TkfM80zJEZLmbfnEC62mMMSaChHVDARImTcMsfzIwHSgGlorIsaq6FxivqjUiUga8ISKrVXVj0AJEZgOzAcaMGUNFRUWcv0L8NDU1Der6DTa2vgYf2yaxGW7rK5HBohoo8YwXAzVh8rynqu3AZhFZjxM8lqlqDYCqbhKRCmAqEBQsVHUBsABg2rRpOn369AR8jfioqKhgMNdvsLH1NfjYNonNcFtfieyGWgZMFpGJIpIGXAaEXtX0HDADQEQKcbqlNolIvoike9JPA9ZijDFmQCSsZaGqHSJyE/AKkAw8rKofishcYLmqLnanzRSRtUAn8F+qWi8inwF+JyI+nIA233sVlTHGmP6VyG4oVPUl4KWQtB97hhX4tvvx5nkHOC6RdTPGGBM9u4PbGGNMRBYsjDHGRGTBwhhjTEQWLIwxxkRkwcIYY0xEFiyMMcZEZMHCGGNMRBYsjDHGRGTBwhhjTEQWLIwxxkRkwcIYY0xEFiyMMcZEZMHCGGNMRBYsjDHGRGTBwhhjTEQWLIwxxkRkwcIYY0xEFiyMMcZEZMHCGGNMRAkNFiJynoisF5ENInJrN3kuFZG1IvKhiPzJk361iHzifq5OZD2NMcb0LCVRBYtIMnA/cA5QDSwTkcWqutaTZzJwG3Caqu4RkdFu+ijgJ8A0QIH33Xn3JKq+xhhjupfIlsXJwAZV3aSqbcAi4KKQPNcB9/uDgKrudNPPBV5T1d3utNeA8xJYV2OMMT1IWMsCGAdUecargVNC8hwJICJvA8nA7ar6cjfzjgtdgIjMBma7o00ish1o6GV9c3sxbyzzFAK7Yiz/UDYe2DrQlYhRb35DQ2nZ8d4m8ahzb8tI9N87DJ2/+dJoMiUyWEiYNA2z/MnAdKAYWCoix0Y5L6q6AFgQWKDIAlWdHZovqsr2Yt5Y5hGR5ao6rTd1OxSJSN1QW199+f0NhWXHe5vEo869LSPRf+9u/mH1N5/IbqhqoMQzXgzUhMnzvKq2q+pmYD1O8Ihm3nBe6H11ezVvX5ZnerZ3oCvQCwP5e+iPZcd7m8Sjzr0tw/7eYySqXQ7Y41OwSArwMXAWsA1YBlyhqh968pwHXK6qV4tIIbACKMc9qQ2c6Gb9F3CSqu5OSGX7wXA7ykg0W1+Dj22T2Ay39ZWwbihV7RCRm4BXcM5HPKyqH4rIXGC5qi52p80UkbVAJ/BfqloPICLzcAIMwNyhHChcCyJnMR62vgYf2yaxGVbrK2EtC2OMMcOH3cFtjDEmIgsWxhhjIrJgkQAi8rCI7BSRNZ60USLymvv4ktdEJH8g6ziYiEiJiCwRkY/cx778p5tu62yAiMgWEVktIitFZLmbZtvDI5a/c3H8yn300SoRObH7kgcnCxaJ8Shd7zi/FXhdVScDr7vjxtEBfEdVpwCfBm4UkaOxdTbQZqhqueeKHtsewR4l+r/z83FuC5iMcyPxb/upjnFjwSIBVPVNIPTqrYuAx9zhx4CL+7VSg5iq1qrqv9zhRuAjnDv2bZ0NLrY9PGL8O78IeFwd7wF5InJY/9Q0PixY9J8xqloLzs4RGD3A9RmURGQCMBX4B7bOBpICr4rI++5jdcC2RzS6W0dRPcJoMEvk4z6MiYmIjASeBuao6j6RcE99Mf3kNFWtcZ8E/ZqIrBvoCg1xUT3CaDCzlkX/2eFvdrr/74yQ/5AiIqk4gWKhqj7jJts6GyCqWuP+vxN4Fucp0rY9IutuHfX2EUaDhgWL/rMY8L/E6Wrg+QGsy6AiThPi98BHqvo/nkm2zgaAiIwQkWz/MDATWINtj2h0t44WA1e5V0V9Gmjwd1cNFXYHdwKIyBM4T9ItBHbgvMjpOeBJDj7m+UvD4BEmcSEipwNLgdWAz03+vzjnLWyd9TMRKcNpTYDTVf0nVb1TRAqw7REQy9+5e0D0a5yrp5qBa1V1+UDUu7csWBhjjInIuqGMMcZEZMHCGGNMRBYsjDHGRGTBwhhjTEQWLIwxxkRkwcIMeyJSISIJf72liNzsPjl3YbT1EZGXRCQv0XULU4e5InJ2fy/XDF32uA9jeiAiKaraEWX2G4DzVXVztOWr6ud6V7O+UdUfD8RyzdBlLQszKIjIBPeo/EH3nRavikimO817JF4oIlvc4WtE5DkReUFENovITSLybRFZISLvicgozyK+IiLviMgaETnZnX+E+06CZe48F3nK/YuIvAC8Gqau33bLWSMic9y0B4AyYLGI3BKSP1NEFrnvMfgzkOmZtsX9ThNEZJ2IPOSWu1BEzhaRt913I0RT52dE5GU3/8/c9GQRedQtc7W/bm7aF93hs9yyVrtlp3vqdoeI/Mud9ik3/bPivOdipTtfdl+2vRkiVNU+9hnwDzAB570W5e74k8BX3OEKYJo7XAhscYevATYA2UAR0ABc7077Jc4DCf3zP+gOnwmscYfv8iwjD/gYGOGWWw2MClPPk3DuNB8BjAQ+BKa607YAhWHm+TbwsDt8vPs9p3nn8Xz/43AO4t4HHsZ5AN1FwHNR1HkTkAtkAJU4zyI6CXjNU5c89/9HgS+6eauAI930xz3rbQvwLXf4BuAhd/gFnAcN4q6DlIH+/dgn8R9rWZjBZLOqrnSH38fZgUayRFUbVbUOJ1i84KavDpn/CQi8gyDHPU8wE7hVRFbiBJQMnMc0gLODDfcoi9OBZ1V1v6o2Ac8AZ0So45nAH93lrwJWdZNvs6quVlUfThB6XVU15Lv0VOfXVbVBVVuAtUApTgApE5H/JyLnAftClnmUu9yP3fHH3Pr6+R/q6N0ebwP/IyI34wSfaLvpzBBmwcIMJq2e4U4OnlPr4OBvNaOHeXyecR/B5+RCn2ujOEftX1DnbXDlqjpeVT9yp+/vpo69fW56NM/Viea79FTnLutPVfcAJ+AElhuBh0KWGen7+MsMbA9VnQ98A6c77T1/95QZ3ixYmKFgC053CjhdJ73xZQg8tLBBVRuAV4BvuQ95Q0SmRlHOm8DFIpLlPpH1EpyHIEaa50p3GcfidEX1Vkx1FpFCIElVnwZ+BIS++3kdMEFEjnDHvwr8PUKZk9wW0E+B5YAFi0OAXQ1lhoJ7gCdF5KvAG70sY4+IvAPkAF9z0+YB9wKr3J3vFmBWT4Wo6r9E5FHgn27SQ6q6IsKyfws8IiKrgJWeeXsj1jqPc5ftPzC8zTtRVVtE5FrgLyKSAiwDHohQhzkiMgOntbEW+N+Yv4UZcuyps8YYYyKybihjjDERWbAwxhgTkQULY4wxEVmwMMYYE5EFC2OMMRFZsDDGGBORBQtjjDER/X8i/2KSEyt/4wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "\n",
    "\n",
    "# with plt.xkcd():\n",
    "xs = np.array([8, 16, 32, 64, 128])\n",
    "ys = aucs_vs_factors\n",
    "axes = plt.axes()\n",
    "plt.semilogx(xs, ys, '-gD', label='BPR')\n",
    "plt.semilogx(xs, aucs_als, '-g^', label='ALS', color='orange')\n",
    "plt.semilogx(xs, aucs_tbpr, '-go', label='tBPR')\n",
    "axes.set_ylim([0.60, 1.0])\n",
    "axes.set_xticks([10, 20, 50, 100])\n",
    "axes.get_xaxis().set_major_formatter(matplotlib.ticker.ScalarFormatter())\n",
    "axes.set_xlabel('number of dimensions')\n",
    "axes.set_ylabel('AUC')\n",
    "plt.title('AUC scores for Reddit January Data')\n",
    "plt.legend()\n",
    "plt.grid()\n",
    "plt.savefig('AUCs.png', dpi=1000, bbox_inches='tight')\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Help on function legend in module matplotlib.pyplot:\n",
      "\n",
      "legend(*args, **kwargs)\n",
      "    Places a legend on the axes.\n",
      "    \n",
      "    Call signatures::\n",
      "    \n",
      "        legend()\n",
      "        legend(labels)\n",
      "        legend(handles, labels)\n",
      "    \n",
      "    The call signatures correspond to three different ways how to use\n",
      "    this method.\n",
      "    \n",
      "    **1. Automatic detection of elements to be shown in the legend**\n",
      "    \n",
      "    The elements to be added to the legend are automatically determined,\n",
      "    when you do not pass in any extra arguments.\n",
      "    \n",
      "    In this case, the labels are taken from the artist. You can specify\n",
      "    them either at artist creation or by calling the\n",
      "    :meth:`~.Artist.set_label` method on the artist::\n",
      "    \n",
      "        line, = ax.plot([1, 2, 3], label='Inline label')\n",
      "        ax.legend()\n",
      "    \n",
      "    or::\n",
      "    \n",
      "        line.set_label('Label via method')\n",
      "        line, = ax.plot([1, 2, 3])\n",
      "        ax.legend()\n",
      "    \n",
      "    Specific lines can be excluded from the automatic legend element\n",
      "    selection by defining a label starting with an underscore.\n",
      "    This is default for all artists, so calling `Axes.legend` without\n",
      "    any arguments and without setting the labels manually will result in\n",
      "    no legend being drawn.\n",
      "    \n",
      "    \n",
      "    **2. Labeling existing plot elements**\n",
      "    \n",
      "    To make a legend for lines which already exist on the axes\n",
      "    (via plot for instance), simply call this function with an iterable\n",
      "    of strings, one for each legend item. For example::\n",
      "    \n",
      "        ax.plot([1, 2, 3])\n",
      "        ax.legend(['A simple line'])\n",
      "    \n",
      "    Note: This way of using is discouraged, because the relation between\n",
      "    plot elements and labels is only implicit by their order and can\n",
      "    easily be mixed up.\n",
      "    \n",
      "    \n",
      "    **3. Explicitly defining the elements in the legend**\n",
      "    \n",
      "    For full control of which artists have a legend entry, it is possible\n",
      "    to pass an iterable of legend artists followed by an iterable of\n",
      "    legend labels respectively::\n",
      "    \n",
      "        legend((line1, line2, line3), ('label1', 'label2', 'label3'))\n",
      "    \n",
      "    Parameters\n",
      "    ----------\n",
      "    \n",
      "    handles : sequence of `.Artist`, optional\n",
      "        A list of Artists (lines, patches) to be added to the legend.\n",
      "        Use this together with *labels*, if you need full control on what\n",
      "        is shown in the legend and the automatic mechanism described above\n",
      "        is not sufficient.\n",
      "    \n",
      "        The length of handles and labels should be the same in this\n",
      "        case. If they are not, they are truncated to the smaller length.\n",
      "    \n",
      "    labels : sequence of strings, optional\n",
      "        A list of labels to show next to the artists.\n",
      "        Use this together with *handles*, if you need full control on what\n",
      "        is shown in the legend and the automatic mechanism described above\n",
      "        is not sufficient.\n",
      "    \n",
      "    Other Parameters\n",
      "    ----------------\n",
      "    \n",
      "    loc : int or string or pair of floats, default: 'upper right'\n",
      "        The location of the legend. Possible codes are:\n",
      "    \n",
      "            ===============   =============\n",
      "            Location String   Location Code\n",
      "            ===============   =============\n",
      "            'best'            0\n",
      "            'upper right'     1\n",
      "            'upper left'      2\n",
      "            'lower left'      3\n",
      "            'lower right'     4\n",
      "            'right'           5\n",
      "            'center left'     6\n",
      "            'center right'    7\n",
      "            'lower center'    8\n",
      "            'upper center'    9\n",
      "            'center'          10\n",
      "            ===============   =============\n",
      "    \n",
      "    \n",
      "        Alternatively can be a 2-tuple giving ``x, y`` of the lower-left\n",
      "        corner of the legend in axes coordinates (in which case\n",
      "        ``bbox_to_anchor`` will be ignored).\n",
      "    \n",
      "    bbox_to_anchor : `.BboxBase` or pair of floats\n",
      "        Specify any arbitrary location for the legend in `bbox_transform`\n",
      "        coordinates (default Axes coordinates).\n",
      "    \n",
      "        For example, to put the legend's upper right hand corner in the\n",
      "        center of the axes the following keywords can be used::\n",
      "    \n",
      "           loc='upper right', bbox_to_anchor=(0.5, 0.5)\n",
      "    \n",
      "    ncol : integer\n",
      "        The number of columns that the legend has. Default is 1.\n",
      "    \n",
      "    prop : None or :class:`matplotlib.font_manager.FontProperties` or dict\n",
      "        The font properties of the legend. If None (default), the current\n",
      "        :data:`matplotlib.rcParams` will be used.\n",
      "    \n",
      "    fontsize : int or float or {'xx-small', 'x-small', 'small', 'medium', 'large', 'x-large', 'xx-large'}\n",
      "        Controls the font size of the legend. If the value is numeric the\n",
      "        size will be the absolute font size in points. String values are\n",
      "        relative to the current default font size. This argument is only\n",
      "        used if `prop` is not specified.\n",
      "    \n",
      "    numpoints : None or int\n",
      "        The number of marker points in the legend when creating a legend\n",
      "        entry for a `.Line2D` (line).\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.numpoints`.\n",
      "    \n",
      "    scatterpoints : None or int\n",
      "        The number of marker points in the legend when creating\n",
      "        a legend entry for a `.PathCollection` (scatter plot).\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.scatterpoints`.\n",
      "    \n",
      "    scatteryoffsets : iterable of floats\n",
      "        The vertical offset (relative to the font size) for the markers\n",
      "        created for a scatter plot legend entry. 0.0 is at the base the\n",
      "        legend text, and 1.0 is at the top. To draw all markers at the\n",
      "        same height, set to ``[0.5]``. Default is ``[0.375, 0.5, 0.3125]``.\n",
      "    \n",
      "    markerscale : None or int or float\n",
      "        The relative size of legend markers compared with the originally\n",
      "        drawn ones.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.markerscale`.\n",
      "    \n",
      "    markerfirst : bool\n",
      "        If *True*, legend marker is placed to the left of the legend label.\n",
      "        If *False*, legend marker is placed to the right of the legend\n",
      "        label.\n",
      "        Default is *True*.\n",
      "    \n",
      "    frameon : None or bool\n",
      "        Control whether the legend should be drawn on a patch\n",
      "        (frame).\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.frameon`.\n",
      "    \n",
      "    fancybox : None or bool\n",
      "        Control whether round edges should be enabled around the\n",
      "        :class:`~matplotlib.patches.FancyBboxPatch` which makes up the\n",
      "        legend's background.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.fancybox`.\n",
      "    \n",
      "    shadow : None or bool\n",
      "        Control whether to draw a shadow behind the legend.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.shadow`.\n",
      "    \n",
      "    framealpha : None or float\n",
      "        Control the alpha transparency of the legend's background.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.framealpha`.  If shadow is activated and\n",
      "        *framealpha* is ``None``, the default value is ignored.\n",
      "    \n",
      "    facecolor : None or \"inherit\" or a color spec\n",
      "        Control the legend's background color.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.facecolor`.  If ``\"inherit\"``, it will take\n",
      "        :rc:`axes.facecolor`.\n",
      "    \n",
      "    edgecolor : None or \"inherit\" or a color spec\n",
      "        Control the legend's background patch edge color.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.edgecolor` If ``\"inherit\"``, it will take\n",
      "        :rc:`axes.edgecolor`.\n",
      "    \n",
      "    mode : {\"expand\", None}\n",
      "        If `mode` is set to ``\"expand\"`` the legend will be horizontally\n",
      "        expanded to fill the axes area (or `bbox_to_anchor` if defines\n",
      "        the legend's size).\n",
      "    \n",
      "    bbox_transform : None or :class:`matplotlib.transforms.Transform`\n",
      "        The transform for the bounding box (`bbox_to_anchor`). For a value\n",
      "        of ``None`` (default) the Axes'\n",
      "        :data:`~matplotlib.axes.Axes.transAxes` transform will be used.\n",
      "    \n",
      "    title : str or None\n",
      "        The legend's title. Default is no title (``None``).\n",
      "    \n",
      "    borderpad : float or None\n",
      "        The fractional whitespace inside the legend border.\n",
      "        Measured in font-size units.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.borderpad`.\n",
      "    \n",
      "    labelspacing : float or None\n",
      "        The vertical space between the legend entries.\n",
      "        Measured in font-size units.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.labelspacing`.\n",
      "    \n",
      "    handlelength : float or None\n",
      "        The length of the legend handles.\n",
      "        Measured in font-size units.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.handlelength`.\n",
      "    \n",
      "    handletextpad : float or None\n",
      "        The pad between the legend handle and text.\n",
      "        Measured in font-size units.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.handletextpad`.\n",
      "    \n",
      "    borderaxespad : float or None\n",
      "        The pad between the axes and legend border.\n",
      "        Measured in font-size units.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.borderaxespad`.\n",
      "    \n",
      "    columnspacing : float or None\n",
      "        The spacing between columns.\n",
      "        Measured in font-size units.\n",
      "        Default is ``None``, which will take the value from\n",
      "        :rc:`legend.columnspacing`.\n",
      "    \n",
      "    handler_map : dict or None\n",
      "        The custom dictionary mapping instances or types to a legend\n",
      "        handler. This `handler_map` updates the default handler map\n",
      "        found at :func:`matplotlib.legend.Legend.get_legend_handler_map`.\n",
      "    \n",
      "    Returns\n",
      "    -------\n",
      "    \n",
      "    :class:`matplotlib.legend.Legend` instance\n",
      "    \n",
      "    Notes\n",
      "    -----\n",
      "    \n",
      "    Not all kinds of artist are supported by the legend command. See\n",
      "    :ref:`sphx_glr_tutorials_intermediate_legend_guide.py` for details.\n",
      "    \n",
      "    Examples\n",
      "    --------\n",
      "    \n",
      "    .. plot:: gallery/api/legend.py\n",
      "\n"
     ]
    }
   ],
   "source": [
    "help(plt.legend)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "@webio": {
   "lastCommId": null,
   "lastKernelId": null
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}