code/03. Brunch Baseline - Collaborative Filtering Recommendation (ALS).ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pickle\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import os, sys, gc \n",
    "from plotnine import *\n",
    "import plotnine\n",
    "\n",
    "from tqdm import tqdm_notebook\n",
    "import seaborn as sns\n",
    "import warnings\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.font_manager as fm\n",
    "import matplotlib as mpl\n",
    "from matplotlib import rc\n",
    "import re\n",
    "from matplotlib.ticker import PercentFormatter\n",
    "import datetime\n",
    "from math import log # IDF 계산을 위해"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "path = 'C:/Users/User/Documents/T아카데미/T 아카데미/input/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.read_json : json 형태의 파일을 dataframe 형태로 불러오는 코드 \n",
    "magazine = pd.read_json(path + 'magazine.json', lines=True) # lines = True : Read the file as a json object per line.\n",
    "metadata = pd.read_json(path + 'metadata.json', lines=True)\n",
    "users = pd.read_json(path + 'users.json', lines=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1eceda8dbeb641d2a6090e698baa074d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3624), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Wall time: 24.5 s\n"
     ]
    }
   ],
   "source": [
    "%%time \n",
    "import itertools\n",
    "from itertools import chain\n",
    "import glob\n",
    "import os \n",
    "\n",
    "input_read_path = path + 'read/read/'\n",
    "# os.listdir : 해당 경로에 있는 모든 파일들을 불러오는 명령어 \n",
    "file_list = os.listdir(input_read_path)\n",
    "exclude_file_lst = ['read.tar', '.2019010120_2019010121.un~']\n",
    "\n",
    "read_df_list = []\n",
    "for file in tqdm_notebook(file_list):\n",
    "    # 예외처리 \n",
    "    if file in exclude_file_lst:\n",
    "        continue \n",
    "    else:\n",
    "        file_path = input_read_path + file\n",
    "        df_temp = pd.read_csv(file_path, header=None, names=['raw'])\n",
    "        # file명을 통해서 읽은 시간을 추출(from, to)\n",
    "        df_temp['from'] = file.split('_')[0]\n",
    "        df_temp['to'] = file.split('_')[1]\n",
    "        read_df_list.append(df_temp)\n",
    "    \n",
    "read_df = pd.concat(read_df_list)\n",
    "# reads 파일을 전처리해서 row 당 user - article이 1:1이 되도록 수정 \n",
    "read_df['user_id'] = read_df['raw'].apply(lambda x: x.split(' ')[0])\n",
    "read_df['article_id'] = read_df['raw'].apply(lambda x: x.split(' ')[1:])\n",
    "\n",
    "def chainer(s):\n",
    "    return list(itertools.chain.from_iterable(s))\n",
    "\n",
    "read_cnt_by_user = read_df['article_id'].map(len)\n",
    "read_rowwise = pd.DataFrame({'from': np.repeat(read_df['from'], read_cnt_by_user),\n",
    "                             'to': np.repeat(read_df['to'], read_cnt_by_user),\n",
    "                             'user_id': np.repeat(read_df['user_id'], read_cnt_by_user),\n",
    "                             'article_id': chainer(read_df['article_id'])})\n",
    "\n",
    "read_rowwise.reset_index(drop=True, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "read_rowwise = pd.read_csv(\"read_rowwise.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from datetime import datetime \n",
    "\n",
    "metadata['reg_datetime'] = metadata['reg_ts'].apply(lambda x : datetime.fromtimestamp(x/1000.0))\n",
    "metadata.loc[metadata['reg_datetime'] == metadata['reg_datetime'].min(), 'reg_datetime'] = datetime(2090, 12, 31)\n",
    "metadata['reg_dt'] = metadata['reg_datetime'].dt.date\n",
    "metadata['type'] = metadata['magazine_id'].apply(lambda x : '개인' if x == 0.0 else '매거진')\n",
    "metadata['reg_dt'] = pd.to_datetime(metadata['reg_dt'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 협업필터링 기반의 모델\n",
    "- Implicit의 ALS 방법"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "read_rowwise = read_rowwise.merge(metadata[['id', 'reg_dt']], how='left', left_on='article_id', right_on='id')\n",
    "read_rowwise = read_rowwise[read_rowwise['article_id'] != '']\n",
    "\n",
    "# 사용자가 읽은 글의 목록들을 저장 \n",
    "read_total = pd.DataFrame(read_rowwise.groupby(['user_id'])['article_id'].unique()).reset_index()\n",
    "read_total.columns = ['user_id', 'article_list']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "read_rowwise = read_rowwise[(read_rowwise['id'].notnull()) & (read_rowwise['reg_dt'].notnull())]\n",
    "read_rowwise = read_rowwise[(read_rowwise['reg_dt'] >= '2019-01-01') & (read_rowwise['reg_dt'] < '2090-12-31')].reset_index(drop=True)\n",
    "del read_rowwise['id']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "user2idx = {}\n",
    "for i, l in enumerate(read_rowwise['user_id'].unique()):\n",
    "    user2idx[l] = i\n",
    "    \n",
    "article2idx = {}\n",
    "for i, l in enumerate(read_rowwise['article_id'].unique()):\n",
    "    article2idx[l] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "148098 48649\n"
     ]
    }
   ],
   "source": [
    "print(len(user2idx), len(article2idx))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "협업필터링을 적용시에 발생가능한 문제점 \n",
    "- 사용자와 아이템의 수가 너무 많기에 메모리상의 문제가 발생\n",
    "- 148098 * 48649\n",
    "\n",
    "해결방안 \n",
    "- 유의미하지 않은 사용자와 아이템을 제거 \n",
    "    - 읽은 article의 갯수가 20개 이하인 user_id\n",
    "    - 사람들에게 읽힌 횟수가 20번 이하인 article_id 는 삭제\n",
    "- 추천할 대상자는 포함 + 추천하지 않은 대상자는 상위 10%만 선택"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "valid = pd.read_csv(path + '/predict/predict/dev.users', header=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": [
    "read_total_valid = read_total[read_total['user_id'].isin(valid[0])].reset_index(drop=True)\n",
    "read_total_train = read_total[~read_total['user_id'].isin(valid[0])].reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "read_total_train['article_len'] = read_total_train['article_list'].apply(lambda x: len(x))\n",
    "top10_percent = np.percentile(read_total_train['article_len'].values, 90)\n",
    "read_total_train = read_total_train[read_total_train['article_len'] >= top10_percent]\n",
    "hot_user = read_total_train['user_id'].unique()\n",
    "\n",
    "user_total = pd.DataFrame(read_rowwise.groupby(['article_id'])['user_id'].unique()).reset_index()\n",
    "user_total.columns = ['article_id', 'user_list']\n",
    "\n",
    "user_total['user_len'] = user_total['user_list'].apply(lambda x: len(x))\n",
    "cold_article = user_total[user_total['user_len'] <= 20]['article_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "read_rowwise = read_rowwise[read_rowwise['user_id'].isin(np.append(hot_user, valid[0].values))]\n",
    "read_rowwise = read_rowwise[~read_rowwise['article_id'].isin(cold_article)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "user2idx = {}\n",
    "for i, l in enumerate(read_rowwise['user_id'].unique()):\n",
    "    user2idx[l] = i\n",
    "    \n",
    "article2idx = {}\n",
    "for i, l in enumerate(read_rowwise['article_id'].unique()):\n",
    "    article2idx[l] = i"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "idx2user = {i: user for user, i in user2idx.items()}\n",
    "idx2article = {i: item for item, i in article2idx.items()}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "29754 20539\n"
     ]
    }
   ],
   "source": [
    "print(len(user2idx), len(article2idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "data = read_rowwise[['user_id', 'article_id']].reset_index(drop=True)\n",
    "useridx = data['useridx'] = read_rowwise['user_id'].apply(lambda x: user2idx[x]).values\n",
    "articleidx = data['articleidx'] = read_rowwise['article_id'].apply(lambda x: article2idx[x]).values\n",
    "rating = np.ones(len(data))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "import scipy\n",
    "\n",
    "purchase_sparse = scipy.sparse.csr_matrix((rating, (useridx, articleidx)), shape=(len(set(useridx)), len(set(articleidx))))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### implicit의 ALS 방법"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "from implicit.evaluation import *\n",
    "from implicit.als import AlternatingLeastSquares as ALS"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|████████████████████████████████████████████████████████████████████████████████| 20.0/20 [00:12<00:00,  1.79it/s]\n"
     ]
    }
   ],
   "source": [
    "als_model = ALS(factors=20, regularization=0.08, iterations = 20)\n",
    "als_model.fit(purchase_sparse.T)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(2477, 0.35188013),\n",
       " (1260, 0.312056),\n",
       " (791, 0.2478193),\n",
       " (5607, 0.23459375),\n",
       " (5598, 0.2330347),\n",
       " (3455, 0.23231715),\n",
       " (6675, 0.23073567),\n",
       " (848, 0.22972026),\n",
       " (6070, 0.22854957),\n",
       " (3440, 0.22490129)]"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "als_model.recommend(0, purchase_sparse, N=150)[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "652f916347594fb38d81edaf18c47e5f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(IntProgress(value=0, max=3000), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "popular_rec_model = read_rowwise['article_id'].value_counts().index[0:1000]\n",
    "\n",
    "with open('./recommend.txt', 'w') as f:\n",
    "    for user in tqdm_notebook(valid[0].values):\n",
    "        # 추천 후보 \n",
    "        seen = chainer(read_total[read_total['user_id'] == user]['article_list'].values)\n",
    "        \n",
    "        try:\n",
    "            recs = als_model.recommend(user2idx[user], purchase_sparse, N=150)\n",
    "            recs = [idx2article[x[0]] for x in recs][0:100]          \n",
    "            f.write('%s %s\\n' % (user, ' '.join(recs)))\n",
    "        except:\n",
    "            recs = popular_rec_model[0:100]\n",
    "            f.write('%s %s\\n' % (user, ' '.join(recs)))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}