Example_code_BPR.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from Models.BPR import BPR\n",
    "from Utils.dataset import implicit_CF_dataset, implicit_CF_dataset_test\n",
    "from Utils.data_utils import read_LOO_settings\n",
    "\n",
    "import torch\n",
    "import torch.utils.data as data\n",
    "import torch.optim as optim\n",
    "\n",
    "from run import LOO_run"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# dummy class\n",
    "class opt:\n",
    "    def __init__(self, max_epoch, early_stop, es_epoch):\n",
    "        self.max_epoch = max_epoch\n",
    "        self.early_stop = early_stop\n",
    "        self.es_epoch = es_epoch"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# gpu setting\n",
    "gpu = torch.device('cuda:' + str(2))\n",
    "\n",
    "# for training\n",
    "model, lr, batch_size, num_ns = 'BPR', 0.001, 1024, 1\n",
    "max_epoch, early_stop, es_epoch= 500, 30, 0\n",
    "reg = 0.001\n",
    "\n",
    "# dataset\n",
    "data_path, dataset, LOO_seed = 'Dataset/', 'citeULike', 0\n",
    "user_count, item_count, train_mat, train_interactions, valid_sample, test_sample, candidates = read_LOO_settings(data_path, dataset, LOO_seed)\n",
    "\n",
    "train_dataset = implicit_CF_dataset(user_count, item_count, train_mat, train_interactions, num_ns)\n",
    "test_dataset = implicit_CF_dataset_test(user_count, test_sample, valid_sample, candidates)\n",
    "\n",
    "train_loader = data.DataLoader(train_dataset, batch_size=batch_size, shuffle=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "User:: 5220 Item:: 25182 Interactions:: 115142\n"
     ]
    }
   ],
   "source": [
    "dim = 100\n",
    "model = BPR(user_count, item_count, dim, gpu)\n",
    "\n",
    "# optimizer\n",
    "model = model.to(gpu)\n",
    "optimizer = optim.Adam(model.parameters(), lr=lr, weight_decay=reg)\n",
    "\n",
    "print(\"User::\", user_count, \"Item::\", item_count, \"Interactions::\", len(train_interactions))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [0/500], Train Loss: 705.6689, Elapsed: train 0.52 test 2.15 *\n",
      "valid H@5: 0.1255, N@5: 0.0904, M@5: 0.0788\n",
      "valid H@10: 0.1705, N@10: 0.1049, M@10: 0.0848\n",
      "valid H@20: 0.2336, N@20: 0.1208, M@20: 0.0891\n",
      "\n",
      "test H@5: 0.1391, N@5: 0.0994, M@5: 0.0864\n",
      "test H@10: 0.1903, N@10: 0.1159, M@10: 0.0931\n",
      "test H@20: 0.2518, N@20: 0.1314, M@20: 0.0973\n",
      "\n",
      "Epoch [1/500], Train Loss: 668.7119, Elapsed: train 0.50 test 1.82 *\n",
      "valid H@5: 0.2152, N@5: 0.1567, M@5: 0.1374\n",
      "valid H@10: 0.2820, N@10: 0.1783, M@10: 0.1462\n",
      "valid H@20: 0.3688, N@20: 0.2001, M@20: 0.1522\n",
      "\n",
      "test H@5: 0.2261, N@5: 0.1679, M@5: 0.1486\n",
      "test H@10: 0.2957, N@10: 0.1904, M@10: 0.1579\n",
      "test H@20: 0.3740, N@20: 0.2102, M@20: 0.1633\n",
      "\n",
      "Epoch [2/500], Train Loss: 544.1658, Elapsed: train 0.50 test 1.93 *\n",
      "valid H@5: 0.2462, N@5: 0.1795, M@5: 0.1575\n",
      "valid H@10: 0.3280, N@10: 0.2058, M@10: 0.1683\n",
      "valid H@20: 0.4053, N@20: 0.2253, M@20: 0.1736\n",
      "\n",
      "test H@5: 0.2522, N@5: 0.1848, M@5: 0.1626\n",
      "test H@10: 0.3292, N@10: 0.2096, M@10: 0.1728\n",
      "test H@20: 0.4156, N@20: 0.2314, M@20: 0.1788\n",
      "\n",
      "Epoch [3/500], Train Loss: 431.6982, Elapsed: train 0.50 test 1.80 *\n",
      "valid H@5: 0.2631, N@5: 0.1939, M@5: 0.1710\n",
      "valid H@10: 0.3491, N@10: 0.2215, M@10: 0.1822\n",
      "valid H@20: 0.4361, N@20: 0.2434, M@20: 0.1882\n",
      "\n",
      "test H@5: 0.2730, N@5: 0.2003, M@5: 0.1763\n",
      "test H@10: 0.3527, N@10: 0.2261, M@10: 0.1869\n",
      "test H@20: 0.4507, N@20: 0.2508, M@20: 0.1937\n",
      "\n",
      "Epoch [4/500], Train Loss: 352.4489, Elapsed: train 0.59 test 2.01 *\n",
      "valid H@5: 0.2786, N@5: 0.2064, M@5: 0.1826\n",
      "valid H@10: 0.3677, N@10: 0.2352, M@10: 0.1944\n",
      "valid H@20: 0.4556, N@20: 0.2573, M@20: 0.2004\n",
      "\n",
      "test H@5: 0.2872, N@5: 0.2126, M@5: 0.1880\n",
      "test H@10: 0.3767, N@10: 0.2416, M@10: 0.2000\n",
      "test H@20: 0.4710, N@20: 0.2654, M@20: 0.2065\n",
      "\n",
      "Epoch [5/500], Train Loss: 296.9626, Elapsed: train 0.53 test 1.91 *\n",
      "valid H@5: 0.2937, N@5: 0.2169, M@5: 0.1917\n",
      "valid H@10: 0.3771, N@10: 0.2438, M@10: 0.2028\n",
      "valid H@20: 0.4710, N@20: 0.2676, M@20: 0.2093\n",
      "\n",
      "test H@5: 0.3006, N@5: 0.2214, M@5: 0.1952\n",
      "test H@10: 0.3913, N@10: 0.2507, M@10: 0.2073\n",
      "test H@20: 0.4896, N@20: 0.2753, M@20: 0.2139\n",
      "\n",
      "Epoch [6/500], Train Loss: 255.0636, Elapsed: train 0.51 test 1.80 *\n",
      "valid H@5: 0.3077, N@5: 0.2258, M@5: 0.1987\n",
      "valid H@10: 0.3939, N@10: 0.2536, M@10: 0.2102\n",
      "valid H@20: 0.4894, N@20: 0.2776, M@20: 0.2167\n",
      "\n",
      "test H@5: 0.3169, N@5: 0.2326, M@5: 0.2049\n",
      "test H@10: 0.4049, N@10: 0.2609, M@10: 0.2164\n",
      "test H@20: 0.5047, N@20: 0.2862, M@20: 0.2234\n",
      "\n",
      "Epoch [7/500], Train Loss: 221.7113, Elapsed: train 0.57 test 1.83 *\n",
      "valid H@5: 0.3206, N@5: 0.2345, M@5: 0.2062\n",
      "valid H@10: 0.4051, N@10: 0.2617, M@10: 0.2173\n",
      "valid H@20: 0.5083, N@20: 0.2878, M@20: 0.2245\n",
      "\n",
      "test H@5: 0.3269, N@5: 0.2423, M@5: 0.2144\n",
      "test H@10: 0.4204, N@10: 0.2724, M@10: 0.2267\n",
      "test H@20: 0.5219, N@20: 0.2979, M@20: 0.2337\n",
      "\n",
      "Epoch [8/500], Train Loss: 193.7541, Elapsed: train 0.45 test 1.81 *\n",
      "valid H@5: 0.3284, N@5: 0.2405, M@5: 0.2115\n",
      "valid H@10: 0.4190, N@10: 0.2698, M@10: 0.2236\n",
      "valid H@20: 0.5216, N@20: 0.2956, M@20: 0.2306\n",
      "\n",
      "test H@5: 0.3363, N@5: 0.2511, M@5: 0.2230\n",
      "test H@10: 0.4305, N@10: 0.2816, M@10: 0.2356\n",
      "test H@20: 0.5348, N@20: 0.3079, M@20: 0.2428\n",
      "\n",
      "Epoch [9/500], Train Loss: 171.9555, Elapsed: train 0.46 test 1.85 *\n",
      "valid H@5: 0.3349, N@5: 0.2461, M@5: 0.2168\n",
      "valid H@10: 0.4304, N@10: 0.2769, M@10: 0.2294\n",
      "valid H@20: 0.5300, N@20: 0.3020, M@20: 0.2363\n",
      "\n",
      "test H@5: 0.3395, N@5: 0.2538, M@5: 0.2255\n",
      "test H@10: 0.4417, N@10: 0.2868, M@10: 0.2391\n",
      "test H@20: 0.5451, N@20: 0.3129, M@20: 0.2462\n",
      "\n",
      "Epoch [10/500], Train Loss: 152.5125, Elapsed: train 0.51 test 1.82 *\n",
      "valid H@5: 0.3430, N@5: 0.2524, M@5: 0.2224\n",
      "valid H@10: 0.4382, N@10: 0.2832, M@10: 0.2352\n",
      "valid H@20: 0.5394, N@20: 0.3088, M@20: 0.2422\n",
      "\n",
      "test H@5: 0.3495, N@5: 0.2616, M@5: 0.2326\n",
      "test H@10: 0.4499, N@10: 0.2941, M@10: 0.2460\n",
      "test H@20: 0.5545, N@20: 0.3205, M@20: 0.2532\n",
      "\n",
      "Epoch [11/500], Train Loss: 138.6231, Elapsed: train 0.46 test 1.85 *\n",
      "valid H@5: 0.3514, N@5: 0.2587, M@5: 0.2281\n",
      "valid H@10: 0.4445, N@10: 0.2889, M@10: 0.2406\n",
      "valid H@20: 0.5522, N@20: 0.3161, M@20: 0.2480\n",
      "\n",
      "test H@5: 0.3593, N@5: 0.2693, M@5: 0.2396\n",
      "test H@10: 0.4593, N@10: 0.3017, M@10: 0.2530\n",
      "test H@20: 0.5624, N@20: 0.3277, M@20: 0.2601\n",
      "\n",
      "Epoch [12/500], Train Loss: 126.5730, Elapsed: train 0.50 test 1.81 *\n",
      "valid H@5: 0.3572, N@5: 0.2640, M@5: 0.2332\n",
      "valid H@10: 0.4541, N@10: 0.2953, M@10: 0.2462\n",
      "valid H@20: 0.5591, N@20: 0.3218, M@20: 0.2534\n",
      "\n",
      "test H@5: 0.3654, N@5: 0.2727, M@5: 0.2421\n",
      "test H@10: 0.4650, N@10: 0.3048, M@10: 0.2553\n",
      "test H@20: 0.5716, N@20: 0.3316, M@20: 0.2626\n",
      "\n",
      "Epoch [13/500], Train Loss: 115.0687, Elapsed: train 0.47 test 1.80 *\n",
      "valid H@5: 0.3665, N@5: 0.2713, M@5: 0.2398\n",
      "valid H@10: 0.4606, N@10: 0.3018, M@10: 0.2524\n",
      "valid H@20: 0.5658, N@20: 0.3282, M@20: 0.2596\n",
      "\n",
      "test H@5: 0.3704, N@5: 0.2768, M@5: 0.2459\n",
      "test H@10: 0.4673, N@10: 0.3084, M@10: 0.2591\n",
      "test H@20: 0.5811, N@20: 0.3371, M@20: 0.2669\n",
      "\n",
      "Epoch [14/500], Train Loss: 105.8583, Elapsed: train 0.50 test 1.72 *\n",
      "valid H@5: 0.3729, N@5: 0.2759, M@5: 0.2439\n",
      "valid H@10: 0.4673, N@10: 0.3063, M@10: 0.2564\n",
      "valid H@20: 0.5727, N@20: 0.3329, M@20: 0.2636\n",
      "\n",
      "test H@5: 0.3788, N@5: 0.2830, M@5: 0.2514\n",
      "test H@10: 0.4773, N@10: 0.3148, M@10: 0.2645\n",
      "test H@20: 0.5877, N@20: 0.3427, M@20: 0.2722\n",
      "\n",
      "Epoch [15/500], Train Loss: 97.8017, Elapsed: train 0.48 test 1.80 *\n",
      "valid H@5: 0.3742, N@5: 0.2797, M@5: 0.2485\n",
      "valid H@10: 0.4725, N@10: 0.3115, M@10: 0.2617\n",
      "valid H@20: 0.5817, N@20: 0.3391, M@20: 0.2692\n",
      "\n",
      "test H@5: 0.3823, N@5: 0.2863, M@5: 0.2546\n",
      "test H@10: 0.4869, N@10: 0.3201, M@10: 0.2686\n",
      "test H@20: 0.5942, N@20: 0.3472, M@20: 0.2760\n",
      "\n",
      "Epoch [16/500], Train Loss: 90.8302, Elapsed: train 0.44 test 1.82 *\n",
      "valid H@5: 0.3744, N@5: 0.2808, M@5: 0.2499\n",
      "valid H@10: 0.4790, N@10: 0.3147, M@10: 0.2639\n",
      "valid H@20: 0.5888, N@20: 0.3424, M@20: 0.2715\n",
      "\n",
      "test H@5: 0.3890, N@5: 0.2896, M@5: 0.2568\n",
      "test H@10: 0.4932, N@10: 0.3233, M@10: 0.2707\n",
      "test H@20: 0.6009, N@20: 0.3504, M@20: 0.2781\n",
      "\n",
      "Epoch [17/500], Train Loss: 84.0110, Elapsed: train 0.52 test 1.95 *\n",
      "valid H@5: 0.3836, N@5: 0.2870, M@5: 0.2551\n",
      "valid H@10: 0.4880, N@10: 0.3206, M@10: 0.2689\n",
      "valid H@20: 0.5923, N@20: 0.3470, M@20: 0.2762\n",
      "\n",
      "test H@5: 0.3941, N@5: 0.2936, M@5: 0.2603\n",
      "test H@10: 0.4940, N@10: 0.3260, M@10: 0.2738\n",
      "test H@20: 0.6059, N@20: 0.3543, M@20: 0.2816\n",
      "\n",
      "Epoch [18/500], Train Loss: 78.9089, Elapsed: train 0.51 test 1.73 *\n",
      "valid H@5: 0.3870, N@5: 0.2895, M@5: 0.2573\n",
      "valid H@10: 0.4961, N@10: 0.3248, M@10: 0.2718\n",
      "valid H@20: 0.5965, N@20: 0.3501, M@20: 0.2787\n",
      "\n",
      "test H@5: 0.3997, N@5: 0.2984, M@5: 0.2648\n",
      "test H@10: 0.5039, N@10: 0.3321, M@10: 0.2788\n",
      "test H@20: 0.6095, N@20: 0.3587, M@20: 0.2861\n",
      "\n",
      "Epoch [19/500], Train Loss: 74.0041, Elapsed: train 0.50 test 1.78 *\n",
      "valid H@5: 0.3949, N@5: 0.2949, M@5: 0.2620\n",
      "valid H@10: 0.4986, N@10: 0.3282, M@10: 0.2756\n",
      "valid H@20: 0.6032, N@20: 0.3546, M@20: 0.2828\n",
      "\n",
      "test H@5: 0.4033, N@5: 0.3009, M@5: 0.2670\n",
      "test H@10: 0.5043, N@10: 0.3336, M@10: 0.2806\n",
      "test H@20: 0.6179, N@20: 0.3624, M@20: 0.2885\n",
      "\n",
      "Epoch [20/500], Train Loss: 68.4981, Elapsed: train 0.51 test 1.83 *\n",
      "valid H@5: 0.3936, N@5: 0.2963, M@5: 0.2642\n",
      "valid H@10: 0.5014, N@10: 0.3311, M@10: 0.2785\n",
      "valid H@20: 0.6089, N@20: 0.3583, M@20: 0.2859\n",
      "\n",
      "test H@5: 0.4028, N@5: 0.3010, M@5: 0.2673\n",
      "test H@10: 0.5137, N@10: 0.3369, M@10: 0.2821\n",
      "test H@20: 0.6225, N@20: 0.3643, M@20: 0.2896\n",
      "\n",
      "Epoch [21/500], Train Loss: 66.0909, Elapsed: train 0.48 test 1.82 *\n",
      "valid H@5: 0.4008, N@5: 0.3016, M@5: 0.2688\n",
      "valid H@10: 0.5041, N@10: 0.3350, M@10: 0.2826\n",
      "valid H@20: 0.6130, N@20: 0.3624, M@20: 0.2901\n",
      "\n",
      "test H@5: 0.4087, N@5: 0.3068, M@5: 0.2731\n",
      "test H@10: 0.5154, N@10: 0.3415, M@10: 0.2875\n",
      "test H@20: 0.6256, N@20: 0.3694, M@20: 0.2952\n",
      "\n",
      "Epoch [22/500], Train Loss: 61.3383, Elapsed: train 0.49 test 1.79 *\n",
      "valid H@5: 0.4049, N@5: 0.3029, M@5: 0.2693\n",
      "valid H@10: 0.5041, N@10: 0.3350, M@10: 0.2825\n",
      "valid H@20: 0.6147, N@20: 0.3631, M@20: 0.2903\n",
      "\n",
      "test H@5: 0.4148, N@5: 0.3103, M@5: 0.2757\n",
      "test H@10: 0.5210, N@10: 0.3446, M@10: 0.2899\n",
      "test H@20: 0.6319, N@20: 0.3727, M@20: 0.2976\n",
      "\n",
      "Epoch [23/500], Train Loss: 58.7653, Elapsed: train 0.43 test 1.80 *\n",
      "valid H@5: 0.4053, N@5: 0.3030, M@5: 0.2693\n",
      "valid H@10: 0.5110, N@10: 0.3373, M@10: 0.2834\n",
      "valid H@20: 0.6195, N@20: 0.3647, M@20: 0.2910\n",
      "\n",
      "test H@5: 0.4189, N@5: 0.3129, M@5: 0.2779\n",
      "test H@10: 0.5244, N@10: 0.3471, M@10: 0.2920\n",
      "test H@20: 0.6354, N@20: 0.3752, M@20: 0.2998\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [24/500], Train Loss: 56.4162, Elapsed: train 0.53 test 1.75 *\n",
      "valid H@5: 0.4074, N@5: 0.3066, M@5: 0.2733\n",
      "valid H@10: 0.5152, N@10: 0.3415, M@10: 0.2877\n",
      "valid H@20: 0.6197, N@20: 0.3680, M@20: 0.2951\n",
      "\n",
      "test H@5: 0.4223, N@5: 0.3162, M@5: 0.2811\n",
      "test H@10: 0.5313, N@10: 0.3514, M@10: 0.2956\n",
      "test H@20: 0.6354, N@20: 0.3778, M@20: 0.3030\n",
      "\n",
      "Epoch [25/500], Train Loss: 53.7824, Elapsed: train 0.49 test 1.76 *\n",
      "valid H@5: 0.4118, N@5: 0.3092, M@5: 0.2753\n",
      "valid H@10: 0.5206, N@10: 0.3444, M@10: 0.2898\n",
      "valid H@20: 0.6227, N@20: 0.3703, M@20: 0.2970\n",
      "\n",
      "test H@5: 0.4229, N@5: 0.3160, M@5: 0.2806\n",
      "test H@10: 0.5317, N@10: 0.3512, M@10: 0.2952\n",
      "test H@20: 0.6392, N@20: 0.3785, M@20: 0.3027\n",
      "\n",
      "Epoch [26/500], Train Loss: 52.7137, Elapsed: train 0.49 test 1.76 *\n",
      "valid H@5: 0.4144, N@5: 0.3101, M@5: 0.2756\n",
      "valid H@10: 0.5242, N@10: 0.3457, M@10: 0.2903\n",
      "valid H@20: 0.6223, N@20: 0.3705, M@20: 0.2972\n",
      "\n",
      "test H@5: 0.4265, N@5: 0.3196, M@5: 0.2843\n",
      "test H@10: 0.5340, N@10: 0.3544, M@10: 0.2986\n",
      "test H@20: 0.6405, N@20: 0.3815, M@20: 0.3061\n",
      "\n",
      "Epoch [27/500], Train Loss: 50.1870, Elapsed: train 0.52 test 1.77 *\n",
      "valid H@5: 0.4187, N@5: 0.3130, M@5: 0.2781\n",
      "valid H@10: 0.5250, N@10: 0.3475, M@10: 0.2924\n",
      "valid H@20: 0.6246, N@20: 0.3729, M@20: 0.2995\n",
      "\n",
      "test H@5: 0.4279, N@5: 0.3220, M@5: 0.2869\n",
      "test H@10: 0.5400, N@10: 0.3581, M@10: 0.3017\n",
      "test H@20: 0.6436, N@20: 0.3843, M@20: 0.3090\n",
      "\n",
      "Epoch [28/500], Train Loss: 47.2426, Elapsed: train 0.46 test 1.76 *\n",
      "valid H@5: 0.4167, N@5: 0.3141, M@5: 0.2802\n",
      "valid H@10: 0.5286, N@10: 0.3504, M@10: 0.2952\n",
      "valid H@20: 0.6294, N@20: 0.3761, M@20: 0.3023\n",
      "\n",
      "test H@5: 0.4309, N@5: 0.3236, M@5: 0.2881\n",
      "test H@10: 0.5398, N@10: 0.3587, M@10: 0.3026\n",
      "test H@20: 0.6465, N@20: 0.3858, M@20: 0.3101\n",
      "\n",
      "Epoch [29/500], Train Loss: 45.8782, Elapsed: train 0.47 test 1.72 *\n",
      "valid H@5: 0.4219, N@5: 0.3181, M@5: 0.2837\n",
      "valid H@10: 0.5315, N@10: 0.3535, M@10: 0.2983\n",
      "valid H@20: 0.6354, N@20: 0.3799, M@20: 0.3056\n",
      "\n",
      "test H@5: 0.4311, N@5: 0.3251, M@5: 0.2900\n",
      "test H@10: 0.5430, N@10: 0.3613, M@10: 0.3051\n",
      "test H@20: 0.6494, N@20: 0.3883, M@20: 0.3125\n",
      "\n",
      "Epoch [30/500], Train Loss: 45.2589, Elapsed: train 0.47 test 1.78 *\n",
      "valid H@5: 0.4246, N@5: 0.3203, M@5: 0.2858\n",
      "valid H@10: 0.5321, N@10: 0.3550, M@10: 0.3001\n",
      "valid H@20: 0.6367, N@20: 0.3816, M@20: 0.3075\n",
      "\n",
      "test H@5: 0.4340, N@5: 0.3271, M@5: 0.2917\n",
      "test H@10: 0.5476, N@10: 0.3637, M@10: 0.3068\n",
      "test H@20: 0.6509, N@20: 0.3899, M@20: 0.3140\n",
      "\n",
      "Epoch [31/500], Train Loss: 43.3200, Elapsed: train 0.46 test 1.86 *\n",
      "valid H@5: 0.4292, N@5: 0.3214, M@5: 0.2858\n",
      "valid H@10: 0.5342, N@10: 0.3553, M@10: 0.2998\n",
      "valid H@20: 0.6369, N@20: 0.3814, M@20: 0.3070\n",
      "\n",
      "test H@5: 0.4355, N@5: 0.3279, M@5: 0.2923\n",
      "test H@10: 0.5486, N@10: 0.3645, M@10: 0.3074\n",
      "test H@20: 0.6517, N@20: 0.3906, M@20: 0.3146\n",
      "\n",
      "Epoch [32/500], Train Loss: 41.4953, Elapsed: train 0.48 test 1.75 *\n",
      "valid H@5: 0.4275, N@5: 0.3223, M@5: 0.2875\n",
      "valid H@10: 0.5361, N@10: 0.3573, M@10: 0.3019\n",
      "valid H@20: 0.6392, N@20: 0.3833, M@20: 0.3090\n",
      "\n",
      "test H@5: 0.4363, N@5: 0.3296, M@5: 0.2942\n",
      "test H@10: 0.5497, N@10: 0.3663, M@10: 0.3094\n",
      "test H@20: 0.6530, N@20: 0.3926, M@20: 0.3167\n",
      "\n",
      "Epoch [33/500], Train Loss: 40.1790, Elapsed: train 0.47 test 1.76 *\n",
      "valid H@5: 0.4325, N@5: 0.3240, M@5: 0.2882\n",
      "valid H@10: 0.5371, N@10: 0.3579, M@10: 0.3022\n",
      "valid H@20: 0.6400, N@20: 0.3839, M@20: 0.3093\n",
      "\n",
      "test H@5: 0.4397, N@5: 0.3319, M@5: 0.2962\n",
      "test H@10: 0.5501, N@10: 0.3677, M@10: 0.3110\n",
      "test H@20: 0.6559, N@20: 0.3945, M@20: 0.3185\n",
      "\n",
      "Epoch [34/500], Train Loss: 39.8134, Elapsed: train 0.56 test 1.78 *\n",
      "valid H@5: 0.4338, N@5: 0.3256, M@5: 0.2898\n",
      "valid H@10: 0.5390, N@10: 0.3596, M@10: 0.3039\n",
      "valid H@20: 0.6436, N@20: 0.3860, M@20: 0.3111\n",
      "\n",
      "test H@5: 0.4401, N@5: 0.3327, M@5: 0.2972\n",
      "test H@10: 0.5532, N@10: 0.3694, M@10: 0.3124\n",
      "test H@20: 0.6557, N@20: 0.3954, M@20: 0.3196\n",
      "\n",
      "Epoch [35/500], Train Loss: 38.3727, Elapsed: train 0.46 test 1.79 *\n",
      "valid H@5: 0.4363, N@5: 0.3279, M@5: 0.2921\n",
      "valid H@10: 0.5388, N@10: 0.3611, M@10: 0.3058\n",
      "valid H@20: 0.6434, N@20: 0.3876, M@20: 0.3131\n",
      "\n",
      "test H@5: 0.4470, N@5: 0.3376, M@5: 0.3014\n",
      "test H@10: 0.5511, N@10: 0.3711, M@10: 0.3152\n",
      "test H@20: 0.6578, N@20: 0.3981, M@20: 0.3226\n",
      "\n",
      "Epoch [36/500], Train Loss: 37.6572, Elapsed: train 0.46 test 1.88 *\n",
      "valid H@5: 0.4403, N@5: 0.3310, M@5: 0.2949\n",
      "valid H@10: 0.5417, N@10: 0.3639, M@10: 0.3085\n",
      "valid H@20: 0.6425, N@20: 0.3895, M@20: 0.3155\n",
      "\n",
      "test H@5: 0.4447, N@5: 0.3373, M@5: 0.3017\n",
      "test H@10: 0.5518, N@10: 0.3719, M@10: 0.3160\n",
      "test H@20: 0.6580, N@20: 0.3988, M@20: 0.3234\n",
      "\n",
      "Epoch [37/500], Train Loss: 35.9071, Elapsed: train 0.50 test 1.85 *\n",
      "valid H@5: 0.4424, N@5: 0.3322, M@5: 0.2957\n",
      "valid H@10: 0.5447, N@10: 0.3653, M@10: 0.3094\n",
      "valid H@20: 0.6446, N@20: 0.3906, M@20: 0.3163\n",
      "\n",
      "test H@5: 0.4463, N@5: 0.3396, M@5: 0.3042\n",
      "test H@10: 0.5555, N@10: 0.3750, M@10: 0.3189\n",
      "test H@20: 0.6589, N@20: 0.4012, M@20: 0.3261\n",
      "\n",
      "Epoch [38/500], Train Loss: 35.5133, Elapsed: train 0.47 test 1.73 *\n",
      "valid H@5: 0.4447, N@5: 0.3339, M@5: 0.2972\n",
      "valid H@10: 0.5486, N@10: 0.3674, M@10: 0.3110\n",
      "valid H@20: 0.6459, N@20: 0.3920, M@20: 0.3177\n",
      "\n",
      "test H@5: 0.4499, N@5: 0.3411, M@5: 0.3051\n",
      "test H@10: 0.5580, N@10: 0.3761, M@10: 0.3195\n",
      "test H@20: 0.6610, N@20: 0.4022, M@20: 0.3267\n",
      "\n",
      "Epoch [39/500], Train Loss: 34.8082, Elapsed: train 0.52 test 1.81 *\n",
      "valid H@5: 0.4443, N@5: 0.3352, M@5: 0.2991\n",
      "valid H@10: 0.5501, N@10: 0.3695, M@10: 0.3133\n",
      "valid H@20: 0.6446, N@20: 0.3934, M@20: 0.3198\n",
      "\n",
      "test H@5: 0.4518, N@5: 0.3428, M@5: 0.3067\n",
      "test H@10: 0.5601, N@10: 0.3779, M@10: 0.3213\n",
      "test H@20: 0.6655, N@20: 0.4045, M@20: 0.3286\n",
      "\n",
      "Epoch [40/500], Train Loss: 33.9425, Elapsed: train 0.52 test 1.80 *\n",
      "valid H@5: 0.4459, N@5: 0.3369, M@5: 0.3008\n",
      "valid H@10: 0.5526, N@10: 0.3715, M@10: 0.3151\n",
      "valid H@20: 0.6488, N@20: 0.3959, M@20: 0.3218\n",
      "\n",
      "test H@5: 0.4516, N@5: 0.3437, M@5: 0.3080\n",
      "test H@10: 0.5601, N@10: 0.3790, M@10: 0.3227\n",
      "test H@20: 0.6670, N@20: 0.4061, M@20: 0.3301\n",
      "\n",
      "Epoch [41/500], Train Loss: 32.5750, Elapsed: train 0.45 test 1.88\n",
      "valid H@5: 0.4459, N@5: 0.3365, M@5: 0.3003\n",
      "valid H@10: 0.5526, N@10: 0.3713, M@10: 0.3148\n",
      "valid H@20: 0.6478, N@20: 0.3955, M@20: 0.3215\n",
      "\n",
      "test H@5: 0.4549, N@5: 0.3457, M@5: 0.3097\n",
      "test H@10: 0.5618, N@10: 0.3805, M@10: 0.3241\n",
      "test H@20: 0.6672, N@20: 0.4070, M@20: 0.3313\n",
      "\n",
      "Epoch [42/500], Train Loss: 32.0401, Elapsed: train 0.43 test 1.80 *\n",
      "valid H@5: 0.4501, N@5: 0.3399, M@5: 0.3035\n",
      "valid H@10: 0.5541, N@10: 0.3738, M@10: 0.3175\n",
      "valid H@20: 0.6474, N@20: 0.3974, M@20: 0.3241\n",
      "\n",
      "test H@5: 0.4555, N@5: 0.3471, M@5: 0.3113\n",
      "test H@10: 0.5654, N@10: 0.3828, M@10: 0.3261\n",
      "test H@20: 0.6678, N@20: 0.4086, M@20: 0.3331\n",
      "\n",
      "Epoch [43/500], Train Loss: 31.8956, Elapsed: train 0.46 test 1.76 *\n",
      "valid H@5: 0.4476, N@5: 0.3384, M@5: 0.3022\n",
      "valid H@10: 0.5564, N@10: 0.3738, M@10: 0.3169\n",
      "valid H@20: 0.6507, N@20: 0.3976, M@20: 0.3235\n",
      "\n",
      "test H@5: 0.4564, N@5: 0.3491, M@5: 0.3136\n",
      "test H@10: 0.5668, N@10: 0.3849, M@10: 0.3284\n",
      "test H@20: 0.6724, N@20: 0.4116, M@20: 0.3357\n",
      "\n",
      "Epoch [44/500], Train Loss: 31.7794, Elapsed: train 0.54 test 1.72 *\n",
      "valid H@5: 0.4530, N@5: 0.3414, M@5: 0.3045\n",
      "valid H@10: 0.5578, N@10: 0.3754, M@10: 0.3186\n",
      "valid H@20: 0.6518, N@20: 0.3993, M@20: 0.3252\n",
      "\n",
      "test H@5: 0.4583, N@5: 0.3504, M@5: 0.3147\n",
      "test H@10: 0.5629, N@10: 0.3842, M@10: 0.3287\n",
      "test H@20: 0.6724, N@20: 0.4120, M@20: 0.3363\n",
      "\n",
      "Epoch [45/500], Train Loss: 30.7565, Elapsed: train 0.52 test 1.82 *\n",
      "valid H@5: 0.4514, N@5: 0.3407, M@5: 0.3040\n",
      "valid H@10: 0.5601, N@10: 0.3759, M@10: 0.3186\n",
      "valid H@20: 0.6543, N@20: 0.3998, M@20: 0.3252\n",
      "\n",
      "test H@5: 0.4578, N@5: 0.3497, M@5: 0.3138\n",
      "test H@10: 0.5674, N@10: 0.3852, M@10: 0.3285\n",
      "test H@20: 0.6743, N@20: 0.4123, M@20: 0.3360\n",
      "\n",
      "Epoch [46/500], Train Loss: 30.3612, Elapsed: train 0.50 test 1.81 *\n",
      "valid H@5: 0.4553, N@5: 0.3413, M@5: 0.3037\n",
      "valid H@10: 0.5599, N@10: 0.3752, M@10: 0.3177\n",
      "valid H@20: 0.6580, N@20: 0.4000, M@20: 0.3245\n",
      "\n",
      "test H@5: 0.4576, N@5: 0.3510, M@5: 0.3156\n",
      "test H@10: 0.5664, N@10: 0.3863, M@10: 0.3303\n",
      "test H@20: 0.6729, N@20: 0.4134, M@20: 0.3378\n",
      "\n",
      "Epoch [47/500], Train Loss: 30.3134, Elapsed: train 0.51 test 1.75 *\n",
      "valid H@5: 0.4528, N@5: 0.3406, M@5: 0.3034\n",
      "valid H@10: 0.5599, N@10: 0.3754, M@10: 0.3179\n",
      "valid H@20: 0.6587, N@20: 0.4005, M@20: 0.3249\n",
      "\n",
      "test H@5: 0.4620, N@5: 0.3528, M@5: 0.3166\n",
      "test H@10: 0.5693, N@10: 0.3874, M@10: 0.3309\n",
      "test H@20: 0.6758, N@20: 0.4144, M@20: 0.3384\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [48/500], Train Loss: 30.2603, Elapsed: train 0.46 test 1.81 *\n",
      "valid H@5: 0.4532, N@5: 0.3423, M@5: 0.3055\n",
      "valid H@10: 0.5597, N@10: 0.3770, M@10: 0.3200\n",
      "valid H@20: 0.6612, N@20: 0.4027, M@20: 0.3271\n",
      "\n",
      "test H@5: 0.4635, N@5: 0.3531, M@5: 0.3166\n",
      "test H@10: 0.5681, N@10: 0.3871, M@10: 0.3306\n",
      "test H@20: 0.6760, N@20: 0.4145, M@20: 0.3383\n",
      "\n",
      "Epoch [49/500], Train Loss: 28.6725, Elapsed: train 0.46 test 1.81 *\n",
      "valid H@5: 0.4530, N@5: 0.3437, M@5: 0.3074\n",
      "valid H@10: 0.5622, N@10: 0.3792, M@10: 0.3222\n",
      "valid H@20: 0.6620, N@20: 0.4045, M@20: 0.3292\n",
      "\n",
      "test H@5: 0.4646, N@5: 0.3537, M@5: 0.3169\n",
      "test H@10: 0.5727, N@10: 0.3886, M@10: 0.3313\n",
      "test H@20: 0.6771, N@20: 0.4152, M@20: 0.3387\n",
      "\n",
      "Epoch [50/500], Train Loss: 28.2332, Elapsed: train 0.53 test 1.64 *\n",
      "valid H@5: 0.4526, N@5: 0.3446, M@5: 0.3088\n",
      "valid H@10: 0.5652, N@10: 0.3811, M@10: 0.3239\n",
      "valid H@20: 0.6609, N@20: 0.4052, M@20: 0.3305\n",
      "\n",
      "test H@5: 0.4654, N@5: 0.3546, M@5: 0.3179\n",
      "test H@10: 0.5735, N@10: 0.3896, M@10: 0.3324\n",
      "test H@20: 0.6779, N@20: 0.4161, M@20: 0.3397\n",
      "\n",
      "Epoch [51/500], Train Loss: 28.1135, Elapsed: train 0.46 test 1.79 *\n",
      "valid H@5: 0.4560, N@5: 0.3466, M@5: 0.3103\n",
      "valid H@10: 0.5652, N@10: 0.3821, M@10: 0.3251\n",
      "valid H@20: 0.6612, N@20: 0.4064, M@20: 0.3318\n",
      "\n",
      "test H@5: 0.4658, N@5: 0.3551, M@5: 0.3185\n",
      "test H@10: 0.5748, N@10: 0.3905, M@10: 0.3332\n",
      "test H@20: 0.6798, N@20: 0.4171, M@20: 0.3405\n",
      "\n",
      "Epoch [52/500], Train Loss: 28.1582, Elapsed: train 0.46 test 1.75 *\n",
      "valid H@5: 0.4570, N@5: 0.3471, M@5: 0.3106\n",
      "valid H@10: 0.5670, N@10: 0.3828, M@10: 0.3254\n",
      "valid H@20: 0.6630, N@20: 0.4070, M@20: 0.3321\n",
      "\n",
      "test H@5: 0.4664, N@5: 0.3564, M@5: 0.3201\n",
      "test H@10: 0.5773, N@10: 0.3923, M@10: 0.3349\n",
      "test H@20: 0.6810, N@20: 0.4185, M@20: 0.3421\n",
      "\n",
      "Epoch [53/500], Train Loss: 27.9709, Elapsed: train 0.49 test 1.93 *\n",
      "valid H@5: 0.4574, N@5: 0.3477, M@5: 0.3113\n",
      "valid H@10: 0.5668, N@10: 0.3830, M@10: 0.3259\n",
      "valid H@20: 0.6655, N@20: 0.4081, M@20: 0.3328\n",
      "\n",
      "test H@5: 0.4669, N@5: 0.3575, M@5: 0.3213\n",
      "test H@10: 0.5787, N@10: 0.3936, M@10: 0.3361\n",
      "test H@20: 0.6825, N@20: 0.4200, M@20: 0.3434\n",
      "\n",
      "Epoch [54/500], Train Loss: 26.8998, Elapsed: train 0.53 test 1.73 *\n",
      "valid H@5: 0.4576, N@5: 0.3485, M@5: 0.3123\n",
      "valid H@10: 0.5656, N@10: 0.3835, M@10: 0.3268\n",
      "valid H@20: 0.6641, N@20: 0.4085, M@20: 0.3337\n",
      "\n",
      "test H@5: 0.4698, N@5: 0.3608, M@5: 0.3248\n",
      "test H@10: 0.5792, N@10: 0.3962, M@10: 0.3394\n",
      "test H@20: 0.6814, N@20: 0.4221, M@20: 0.3465\n",
      "\n",
      "Epoch [55/500], Train Loss: 26.7975, Elapsed: train 0.46 test 1.72 *\n",
      "valid H@5: 0.4558, N@5: 0.3473, M@5: 0.3112\n",
      "valid H@10: 0.5672, N@10: 0.3834, M@10: 0.3263\n",
      "valid H@20: 0.6641, N@20: 0.4081, M@20: 0.3331\n",
      "\n",
      "test H@5: 0.4721, N@5: 0.3598, M@5: 0.3228\n",
      "test H@10: 0.5779, N@10: 0.3941, M@10: 0.3370\n",
      "test H@20: 0.6789, N@20: 0.4198, M@20: 0.3441\n",
      "\n",
      "Epoch [56/500], Train Loss: 26.6319, Elapsed: train 0.47 test 1.79 *\n",
      "valid H@5: 0.4600, N@5: 0.3487, M@5: 0.3119\n",
      "valid H@10: 0.5685, N@10: 0.3840, M@10: 0.3265\n",
      "valid H@20: 0.6668, N@20: 0.4088, M@20: 0.3333\n",
      "\n",
      "test H@5: 0.4742, N@5: 0.3597, M@5: 0.3219\n",
      "test H@10: 0.5825, N@10: 0.3947, M@10: 0.3363\n",
      "test H@20: 0.6789, N@20: 0.4191, M@20: 0.3431\n",
      "\n",
      "Epoch [57/500], Train Loss: 26.0688, Elapsed: train 0.49 test 1.75 *\n",
      "valid H@5: 0.4604, N@5: 0.3504, M@5: 0.3140\n",
      "valid H@10: 0.5712, N@10: 0.3864, M@10: 0.3290\n",
      "valid H@20: 0.6651, N@20: 0.4102, M@20: 0.3355\n",
      "\n",
      "test H@5: 0.4746, N@5: 0.3614, M@5: 0.3241\n",
      "test H@10: 0.5842, N@10: 0.3968, M@10: 0.3387\n",
      "test H@20: 0.6783, N@20: 0.4207, M@20: 0.3452\n",
      "\n",
      "Epoch [58/500], Train Loss: 25.9515, Elapsed: train 0.46 test 1.75 *\n",
      "valid H@5: 0.4639, N@5: 0.3532, M@5: 0.3165\n",
      "valid H@10: 0.5712, N@10: 0.3879, M@10: 0.3309\n",
      "valid H@20: 0.6668, N@20: 0.4121, M@20: 0.3375\n",
      "\n",
      "test H@5: 0.4760, N@5: 0.3638, M@5: 0.3268\n",
      "test H@10: 0.5840, N@10: 0.3988, M@10: 0.3413\n",
      "test H@20: 0.6802, N@20: 0.4231, M@20: 0.3479\n",
      "\n",
      "Epoch [59/500], Train Loss: 25.9783, Elapsed: train 0.47 test 1.75 *\n",
      "valid H@5: 0.4625, N@5: 0.3537, M@5: 0.3176\n",
      "valid H@10: 0.5708, N@10: 0.3889, M@10: 0.3322\n",
      "valid H@20: 0.6701, N@20: 0.4140, M@20: 0.3391\n",
      "\n",
      "test H@5: 0.4752, N@5: 0.3627, M@5: 0.3255\n",
      "test H@10: 0.5829, N@10: 0.3976, M@10: 0.3400\n",
      "test H@20: 0.6827, N@20: 0.4228, M@20: 0.3469\n",
      "\n",
      "Epoch [60/500], Train Loss: 25.4123, Elapsed: train 0.54 test 1.85 *\n",
      "valid H@5: 0.4679, N@5: 0.3549, M@5: 0.3175\n",
      "valid H@10: 0.5710, N@10: 0.3882, M@10: 0.3313\n",
      "valid H@20: 0.6699, N@20: 0.4132, M@20: 0.3381\n",
      "\n",
      "test H@5: 0.4765, N@5: 0.3640, M@5: 0.3268\n",
      "test H@10: 0.5871, N@10: 0.3997, M@10: 0.3415\n",
      "test H@20: 0.6856, N@20: 0.4245, M@20: 0.3483\n",
      "\n",
      "Epoch [61/500], Train Loss: 25.0899, Elapsed: train 0.48 test 1.74 *\n",
      "valid H@5: 0.4658, N@5: 0.3552, M@5: 0.3186\n",
      "valid H@10: 0.5723, N@10: 0.3897, M@10: 0.3328\n",
      "valid H@20: 0.6706, N@20: 0.4146, M@20: 0.3397\n",
      "\n",
      "test H@5: 0.4781, N@5: 0.3657, M@5: 0.3286\n",
      "test H@10: 0.5879, N@10: 0.4011, M@10: 0.3432\n",
      "test H@20: 0.6848, N@20: 0.4256, M@20: 0.3499\n",
      "\n",
      "Epoch [62/500], Train Loss: 24.9254, Elapsed: train 0.54 test 1.77 *\n",
      "valid H@5: 0.4666, N@5: 0.3561, M@5: 0.3196\n",
      "valid H@10: 0.5748, N@10: 0.3912, M@10: 0.3341\n",
      "valid H@20: 0.6699, N@20: 0.4153, M@20: 0.3407\n",
      "\n",
      "test H@5: 0.4760, N@5: 0.3659, M@5: 0.3294\n",
      "test H@10: 0.5879, N@10: 0.4022, M@10: 0.3445\n",
      "test H@20: 0.6861, N@20: 0.4271, M@20: 0.3513\n",
      "\n",
      "Epoch [63/500], Train Loss: 24.8598, Elapsed: train 0.47 test 1.81 *\n",
      "valid H@5: 0.4679, N@5: 0.3558, M@5: 0.3186\n",
      "valid H@10: 0.5752, N@10: 0.3904, M@10: 0.3329\n",
      "valid H@20: 0.6733, N@20: 0.4152, M@20: 0.3397\n",
      "\n",
      "test H@5: 0.4804, N@5: 0.3671, M@5: 0.3298\n",
      "test H@10: 0.5890, N@10: 0.4023, M@10: 0.3443\n",
      "test H@20: 0.6884, N@20: 0.4274, M@20: 0.3512\n",
      "\n",
      "Epoch [64/500], Train Loss: 25.2640, Elapsed: train 0.49 test 1.82 *\n",
      "valid H@5: 0.4685, N@5: 0.3567, M@5: 0.3196\n",
      "valid H@10: 0.5769, N@10: 0.3917, M@10: 0.3341\n",
      "valid H@20: 0.6722, N@20: 0.4158, M@20: 0.3407\n",
      "\n",
      "test H@5: 0.4794, N@5: 0.3674, M@5: 0.3304\n",
      "test H@10: 0.5902, N@10: 0.4032, M@10: 0.3451\n",
      "test H@20: 0.6894, N@20: 0.4284, M@20: 0.3521\n",
      "\n",
      "Epoch [65/500], Train Loss: 24.1626, Elapsed: train 0.47 test 1.93 *\n",
      "valid H@5: 0.4692, N@5: 0.3588, M@5: 0.3222\n",
      "valid H@10: 0.5758, N@10: 0.3933, M@10: 0.3365\n",
      "valid H@20: 0.6697, N@20: 0.4171, M@20: 0.3430\n",
      "\n",
      "test H@5: 0.4792, N@5: 0.3682, M@5: 0.3314\n",
      "test H@10: 0.5903, N@10: 0.4042, M@10: 0.3464\n",
      "test H@20: 0.6896, N@20: 0.4294, M@20: 0.3533\n",
      "\n",
      "Epoch [66/500], Train Loss: 23.9502, Elapsed: train 0.51 test 1.89 *\n",
      "valid H@5: 0.4700, N@5: 0.3588, M@5: 0.3219\n",
      "valid H@10: 0.5823, N@10: 0.3952, M@10: 0.3370\n",
      "valid H@20: 0.6712, N@20: 0.4175, M@20: 0.3430\n",
      "\n",
      "test H@5: 0.4792, N@5: 0.3679, M@5: 0.3310\n",
      "test H@10: 0.5880, N@10: 0.4033, M@10: 0.3458\n",
      "test H@20: 0.6915, N@20: 0.4295, M@20: 0.3530\n",
      "\n",
      "Epoch [67/500], Train Loss: 24.0586, Elapsed: train 0.53 test 1.74\n",
      "valid H@5: 0.4691, N@5: 0.3588, M@5: 0.3222\n",
      "valid H@10: 0.5792, N@10: 0.3945, M@10: 0.3371\n",
      "valid H@20: 0.6699, N@20: 0.4175, M@20: 0.3434\n",
      "\n",
      "test H@5: 0.4817, N@5: 0.3705, M@5: 0.3338\n",
      "test H@10: 0.5911, N@10: 0.4060, M@10: 0.3485\n",
      "test H@20: 0.6892, N@20: 0.4309, M@20: 0.3554\n",
      "\n",
      "Epoch [68/500], Train Loss: 23.2300, Elapsed: train 0.48 test 1.74 *\n",
      "valid H@5: 0.4714, N@5: 0.3609, M@5: 0.3244\n",
      "valid H@10: 0.5783, N@10: 0.3957, M@10: 0.3388\n",
      "valid H@20: 0.6716, N@20: 0.4192, M@20: 0.3453\n",
      "\n",
      "test H@5: 0.4840, N@5: 0.3717, M@5: 0.3347\n",
      "test H@10: 0.5926, N@10: 0.4069, M@10: 0.3492\n",
      "test H@20: 0.6906, N@20: 0.4317, M@20: 0.3561\n",
      "\n",
      "Epoch [69/500], Train Loss: 23.3157, Elapsed: train 0.47 test 1.82 *\n",
      "valid H@5: 0.4731, N@5: 0.3615, M@5: 0.3246\n",
      "valid H@10: 0.5794, N@10: 0.3961, M@10: 0.3389\n",
      "valid H@20: 0.6735, N@20: 0.4198, M@20: 0.3454\n",
      "\n",
      "test H@5: 0.4840, N@5: 0.3723, M@5: 0.3354\n",
      "test H@10: 0.5913, N@10: 0.4070, M@10: 0.3497\n",
      "test H@20: 0.6902, N@20: 0.4321, M@20: 0.3567\n",
      "\n",
      "Epoch [70/500], Train Loss: 23.3396, Elapsed: train 0.53 test 1.78 *\n",
      "valid H@5: 0.4721, N@5: 0.3618, M@5: 0.3252\n",
      "valid H@10: 0.5785, N@10: 0.3965, M@10: 0.3397\n",
      "valid H@20: 0.6750, N@20: 0.4209, M@20: 0.3464\n",
      "\n",
      "test H@5: 0.4806, N@5: 0.3707, M@5: 0.3343\n",
      "test H@10: 0.5938, N@10: 0.4074, M@10: 0.3495\n",
      "test H@20: 0.6915, N@20: 0.4322, M@20: 0.3564\n",
      "\n",
      "Epoch [71/500], Train Loss: 23.1356, Elapsed: train 0.47 test 1.79 *\n",
      "valid H@5: 0.4717, N@5: 0.3613, M@5: 0.3246\n",
      "valid H@10: 0.5787, N@10: 0.3961, M@10: 0.3392\n",
      "valid H@20: 0.6752, N@20: 0.4205, M@20: 0.3459\n",
      "\n",
      "test H@5: 0.4809, N@5: 0.3706, M@5: 0.3340\n",
      "test H@10: 0.5967, N@10: 0.4079, M@10: 0.3494\n",
      "test H@20: 0.6944, N@20: 0.4326, M@20: 0.3562\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [72/500], Train Loss: 22.9766, Elapsed: train 0.43 test 1.75 *\n",
      "valid H@5: 0.4775, N@5: 0.3625, M@5: 0.3245\n",
      "valid H@10: 0.5823, N@10: 0.3965, M@10: 0.3386\n",
      "valid H@20: 0.6762, N@20: 0.4202, M@20: 0.3451\n",
      "\n",
      "test H@5: 0.4853, N@5: 0.3706, M@5: 0.3327\n",
      "test H@10: 0.5940, N@10: 0.4057, M@10: 0.3472\n",
      "test H@20: 0.6925, N@20: 0.4307, M@20: 0.3541\n",
      "\n",
      "Epoch [73/500], Train Loss: 22.7681, Elapsed: train 0.49 test 1.72 *\n",
      "valid H@5: 0.4779, N@5: 0.3627, M@5: 0.3245\n",
      "valid H@10: 0.5806, N@10: 0.3960, M@10: 0.3384\n",
      "valid H@20: 0.6768, N@20: 0.4204, M@20: 0.3451\n",
      "\n",
      "test H@5: 0.4850, N@5: 0.3725, M@5: 0.3353\n",
      "test H@10: 0.5913, N@10: 0.4069, M@10: 0.3495\n",
      "test H@20: 0.6929, N@20: 0.4327, M@20: 0.3567\n",
      "\n",
      "Epoch [74/500], Train Loss: 22.6391, Elapsed: train 0.53 test 1.74 *\n",
      "valid H@5: 0.4788, N@5: 0.3633, M@5: 0.3250\n",
      "valid H@10: 0.5827, N@10: 0.3970, M@10: 0.3390\n",
      "valid H@20: 0.6781, N@20: 0.4212, M@20: 0.3457\n",
      "\n",
      "test H@5: 0.4834, N@5: 0.3723, M@5: 0.3356\n",
      "test H@10: 0.5928, N@10: 0.4077, M@10: 0.3502\n",
      "test H@20: 0.6948, N@20: 0.4335, M@20: 0.3572\n",
      "\n",
      "Epoch [75/500], Train Loss: 22.3696, Elapsed: train 0.49 test 1.79 *\n",
      "valid H@5: 0.4769, N@5: 0.3643, M@5: 0.3269\n",
      "valid H@10: 0.5827, N@10: 0.3988, M@10: 0.3413\n",
      "valid H@20: 0.6791, N@20: 0.4232, M@20: 0.3481\n",
      "\n",
      "test H@5: 0.4830, N@5: 0.3729, M@5: 0.3364\n",
      "test H@10: 0.5919, N@10: 0.4082, M@10: 0.3511\n",
      "test H@20: 0.6938, N@20: 0.4340, M@20: 0.3582\n",
      "\n",
      "Epoch [76/500], Train Loss: 22.6316, Elapsed: train 0.49 test 1.79 *\n",
      "valid H@5: 0.4806, N@5: 0.3648, M@5: 0.3265\n",
      "valid H@10: 0.5840, N@10: 0.3983, M@10: 0.3403\n",
      "valid H@20: 0.6791, N@20: 0.4223, M@20: 0.3469\n",
      "\n",
      "test H@5: 0.4829, N@5: 0.3733, M@5: 0.3370\n",
      "test H@10: 0.5926, N@10: 0.4088, M@10: 0.3517\n",
      "test H@20: 0.6917, N@20: 0.4339, M@20: 0.3586\n",
      "\n",
      "Epoch [77/500], Train Loss: 22.5067, Elapsed: train 0.49 test 1.81 *\n",
      "valid H@5: 0.4775, N@5: 0.3645, M@5: 0.3270\n",
      "valid H@10: 0.5846, N@10: 0.3993, M@10: 0.3414\n",
      "valid H@20: 0.6794, N@20: 0.4232, M@20: 0.3479\n",
      "\n",
      "test H@5: 0.4819, N@5: 0.3737, M@5: 0.3379\n",
      "test H@10: 0.5934, N@10: 0.4098, M@10: 0.3528\n",
      "test H@20: 0.6925, N@20: 0.4348, M@20: 0.3596\n",
      "\n",
      "Epoch [78/500], Train Loss: 21.7116, Elapsed: train 0.50 test 1.79 *\n",
      "valid H@5: 0.4784, N@5: 0.3652, M@5: 0.3276\n",
      "valid H@10: 0.5863, N@10: 0.4002, M@10: 0.3422\n",
      "valid H@20: 0.6827, N@20: 0.4245, M@20: 0.3488\n",
      "\n",
      "test H@5: 0.4802, N@5: 0.3727, M@5: 0.3371\n",
      "test H@10: 0.5913, N@10: 0.4089, M@10: 0.3522\n",
      "test H@20: 0.6942, N@20: 0.4349, M@20: 0.3593\n",
      "\n",
      "Epoch [79/500], Train Loss: 21.7727, Elapsed: train 0.49 test 1.77\n",
      "valid H@5: 0.4796, N@5: 0.3661, M@5: 0.3285\n",
      "valid H@10: 0.5857, N@10: 0.4006, M@10: 0.3428\n",
      "valid H@20: 0.6802, N@20: 0.4245, M@20: 0.3493\n",
      "\n",
      "test H@5: 0.4830, N@5: 0.3750, M@5: 0.3392\n",
      "test H@10: 0.5923, N@10: 0.4105, M@10: 0.3540\n",
      "test H@20: 0.6971, N@20: 0.4370, M@20: 0.3613\n",
      "\n",
      "Epoch [80/500], Train Loss: 21.6845, Elapsed: train 0.52 test 1.84\n",
      "valid H@5: 0.4802, N@5: 0.3674, M@5: 0.3300\n",
      "valid H@10: 0.5859, N@10: 0.4016, M@10: 0.3442\n",
      "valid H@20: 0.6804, N@20: 0.4255, M@20: 0.3507\n",
      "\n",
      "test H@5: 0.4894, N@5: 0.3765, M@5: 0.3392\n",
      "test H@10: 0.5917, N@10: 0.4097, M@10: 0.3530\n",
      "test H@20: 0.6988, N@20: 0.4368, M@20: 0.3604\n",
      "\n",
      "Epoch [81/500], Train Loss: 21.4633, Elapsed: train 0.50 test 1.91 *\n",
      "valid H@5: 0.4829, N@5: 0.3675, M@5: 0.3293\n",
      "valid H@10: 0.5848, N@10: 0.4006, M@10: 0.3430\n",
      "valid H@20: 0.6821, N@20: 0.4253, M@20: 0.3498\n",
      "\n",
      "test H@5: 0.4890, N@5: 0.3770, M@5: 0.3400\n",
      "test H@10: 0.5949, N@10: 0.4114, M@10: 0.3543\n",
      "test H@20: 0.6982, N@20: 0.4375, M@20: 0.3615\n",
      "\n",
      "Epoch [82/500], Train Loss: 21.6526, Elapsed: train 0.53 test 1.76 *\n",
      "valid H@5: 0.4823, N@5: 0.3683, M@5: 0.3305\n",
      "valid H@10: 0.5859, N@10: 0.4020, M@10: 0.3445\n",
      "valid H@20: 0.6846, N@20: 0.4269, M@20: 0.3513\n",
      "\n",
      "test H@5: 0.4896, N@5: 0.3765, M@5: 0.3391\n",
      "test H@10: 0.5953, N@10: 0.4108, M@10: 0.3533\n",
      "test H@20: 0.6984, N@20: 0.4369, M@20: 0.3605\n",
      "\n",
      "Epoch [83/500], Train Loss: 21.3122, Elapsed: train 0.54 test 1.77 *\n",
      "valid H@5: 0.4840, N@5: 0.3687, M@5: 0.3305\n",
      "valid H@10: 0.5856, N@10: 0.4018, M@10: 0.3443\n",
      "valid H@20: 0.6837, N@20: 0.4266, M@20: 0.3511\n",
      "\n",
      "test H@5: 0.4938, N@5: 0.3794, M@5: 0.3416\n",
      "test H@10: 0.5967, N@10: 0.4127, M@10: 0.3554\n",
      "test H@20: 0.6982, N@20: 0.4384, M@20: 0.3624\n",
      "\n",
      "Epoch [84/500], Train Loss: 21.4271, Elapsed: train 0.54 test 1.84 *\n",
      "valid H@5: 0.4825, N@5: 0.3691, M@5: 0.3315\n",
      "valid H@10: 0.5865, N@10: 0.4031, M@10: 0.3457\n",
      "valid H@20: 0.6856, N@20: 0.4281, M@20: 0.3526\n",
      "\n",
      "test H@5: 0.4932, N@5: 0.3797, M@5: 0.3421\n",
      "test H@10: 0.5988, N@10: 0.4140, M@10: 0.3564\n",
      "test H@20: 0.6982, N@20: 0.4392, M@20: 0.3633\n",
      "\n",
      "Epoch [85/500], Train Loss: 20.9995, Elapsed: train 0.47 test 1.84 *\n",
      "valid H@5: 0.4842, N@5: 0.3693, M@5: 0.3312\n",
      "valid H@10: 0.5890, N@10: 0.4033, M@10: 0.3452\n",
      "valid H@20: 0.6856, N@20: 0.4276, M@20: 0.3519\n",
      "\n",
      "test H@5: 0.4922, N@5: 0.3797, M@5: 0.3424\n",
      "test H@10: 0.5990, N@10: 0.4143, M@10: 0.3568\n",
      "test H@20: 0.6990, N@20: 0.4397, M@20: 0.3638\n",
      "\n",
      "Epoch [86/500], Train Loss: 21.4366, Elapsed: train 0.46 test 1.79 *\n",
      "valid H@5: 0.4848, N@5: 0.3702, M@5: 0.3322\n",
      "valid H@10: 0.5884, N@10: 0.4038, M@10: 0.3461\n",
      "valid H@20: 0.6848, N@20: 0.4281, M@20: 0.3527\n",
      "\n",
      "test H@5: 0.4940, N@5: 0.3809, M@5: 0.3435\n",
      "test H@10: 0.6007, N@10: 0.4153, M@10: 0.3577\n",
      "test H@20: 0.7001, N@20: 0.4405, M@20: 0.3646\n",
      "\n",
      "Epoch [87/500], Train Loss: 21.0033, Elapsed: train 0.51 test 1.81 *\n",
      "valid H@5: 0.4865, N@5: 0.3713, M@5: 0.3331\n",
      "valid H@10: 0.5875, N@10: 0.4041, M@10: 0.3467\n",
      "valid H@20: 0.6844, N@20: 0.4287, M@20: 0.3535\n",
      "\n",
      "test H@5: 0.4940, N@5: 0.3802, M@5: 0.3425\n",
      "test H@10: 0.5988, N@10: 0.4142, M@10: 0.3567\n",
      "test H@20: 0.7005, N@20: 0.4401, M@20: 0.3639\n",
      "\n",
      "Epoch [88/500], Train Loss: 21.3288, Elapsed: train 0.50 test 1.79 *\n",
      "valid H@5: 0.4865, N@5: 0.3719, M@5: 0.3338\n",
      "valid H@10: 0.5896, N@10: 0.4052, M@10: 0.3475\n",
      "valid H@20: 0.6858, N@20: 0.4295, M@20: 0.3542\n",
      "\n",
      "test H@5: 0.4903, N@5: 0.3795, M@5: 0.3427\n",
      "test H@10: 0.6024, N@10: 0.4158, M@10: 0.3578\n",
      "test H@20: 0.7015, N@20: 0.4408, M@20: 0.3646\n",
      "\n",
      "Epoch [89/500], Train Loss: 21.0324, Elapsed: train 0.49 test 1.91 *\n",
      "valid H@5: 0.4838, N@5: 0.3710, M@5: 0.3335\n",
      "valid H@10: 0.5894, N@10: 0.4052, M@10: 0.3477\n",
      "valid H@20: 0.6873, N@20: 0.4300, M@20: 0.3545\n",
      "\n",
      "test H@5: 0.4926, N@5: 0.3790, M@5: 0.3413\n",
      "test H@10: 0.5982, N@10: 0.4132, M@10: 0.3555\n",
      "test H@20: 0.7015, N@20: 0.4395, M@20: 0.3628\n",
      "\n",
      "Epoch [90/500], Train Loss: 20.3906, Elapsed: train 0.53 test 1.80 *\n",
      "valid H@5: 0.4863, N@5: 0.3710, M@5: 0.3328\n",
      "valid H@10: 0.5905, N@10: 0.4048, M@10: 0.3468\n",
      "valid H@20: 0.6856, N@20: 0.4289, M@20: 0.3534\n",
      "\n",
      "test H@5: 0.4936, N@5: 0.3798, M@5: 0.3421\n",
      "test H@10: 0.6005, N@10: 0.4144, M@10: 0.3563\n",
      "test H@20: 0.7017, N@20: 0.4401, M@20: 0.3634\n",
      "\n",
      "Epoch [91/500], Train Loss: 20.7061, Elapsed: train 0.46 test 1.78 *\n",
      "valid H@5: 0.4829, N@5: 0.3710, M@5: 0.3338\n",
      "valid H@10: 0.5903, N@10: 0.4059, M@10: 0.3484\n",
      "valid H@20: 0.6883, N@20: 0.4307, M@20: 0.3551\n",
      "\n",
      "test H@5: 0.4940, N@5: 0.3804, M@5: 0.3428\n",
      "test H@10: 0.6045, N@10: 0.4161, M@10: 0.3575\n",
      "test H@20: 0.7005, N@20: 0.4405, M@20: 0.3642\n",
      "\n",
      "Epoch [92/500], Train Loss: 20.6119, Elapsed: train 0.47 test 1.76 *\n",
      "valid H@5: 0.4861, N@5: 0.3722, M@5: 0.3344\n",
      "valid H@10: 0.5905, N@10: 0.4062, M@10: 0.3485\n",
      "valid H@20: 0.6886, N@20: 0.4309, M@20: 0.3553\n",
      "\n",
      "test H@5: 0.4928, N@5: 0.3792, M@5: 0.3416\n",
      "test H@10: 0.6030, N@10: 0.4150, M@10: 0.3564\n",
      "test H@20: 0.7013, N@20: 0.4399, M@20: 0.3633\n",
      "\n",
      "Epoch [93/500], Train Loss: 20.2297, Elapsed: train 0.49 test 1.79 *\n",
      "valid H@5: 0.4876, N@5: 0.3740, M@5: 0.3363\n",
      "valid H@10: 0.5928, N@10: 0.4080, M@10: 0.3504\n",
      "valid H@20: 0.6894, N@20: 0.4324, M@20: 0.3570\n",
      "\n",
      "test H@5: 0.4922, N@5: 0.3793, M@5: 0.3418\n",
      "test H@10: 0.6043, N@10: 0.4155, M@10: 0.3567\n",
      "test H@20: 0.7032, N@20: 0.4406, M@20: 0.3637\n",
      "\n",
      "Epoch [94/500], Train Loss: 20.3947, Elapsed: train 0.53 test 1.88 *\n",
      "valid H@5: 0.4865, N@5: 0.3727, M@5: 0.3350\n",
      "valid H@10: 0.5907, N@10: 0.4067, M@10: 0.3492\n",
      "valid H@20: 0.6904, N@20: 0.4317, M@20: 0.3560\n",
      "\n",
      "test H@5: 0.4924, N@5: 0.3808, M@5: 0.3438\n",
      "test H@10: 0.6034, N@10: 0.4167, M@10: 0.3586\n",
      "test H@20: 0.7009, N@20: 0.4414, M@20: 0.3654\n",
      "\n",
      "Epoch [95/500], Train Loss: 19.9682, Elapsed: train 0.49 test 1.87\n",
      "valid H@5: 0.4863, N@5: 0.3743, M@5: 0.3371\n",
      "valid H@10: 0.5913, N@10: 0.4084, M@10: 0.3513\n",
      "valid H@20: 0.6879, N@20: 0.4328, M@20: 0.3580\n",
      "\n",
      "test H@5: 0.4940, N@5: 0.3821, M@5: 0.3450\n",
      "test H@10: 0.6032, N@10: 0.4174, M@10: 0.3596\n",
      "test H@20: 0.7024, N@20: 0.4426, M@20: 0.3665\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [96/500], Train Loss: 20.1196, Elapsed: train 0.50 test 1.72 *\n",
      "valid H@5: 0.4873, N@5: 0.3748, M@5: 0.3374\n",
      "valid H@10: 0.5930, N@10: 0.4090, M@10: 0.3516\n",
      "valid H@20: 0.6860, N@20: 0.4325, M@20: 0.3581\n",
      "\n",
      "test H@5: 0.4957, N@5: 0.3823, M@5: 0.3448\n",
      "test H@10: 0.6026, N@10: 0.4167, M@10: 0.3589\n",
      "test H@20: 0.7003, N@20: 0.4416, M@20: 0.3658\n",
      "\n",
      "Epoch [97/500], Train Loss: 20.0154, Elapsed: train 0.55 test 1.70\n",
      "valid H@5: 0.4873, N@5: 0.3748, M@5: 0.3374\n",
      "valid H@10: 0.5919, N@10: 0.4087, M@10: 0.3515\n",
      "valid H@20: 0.6873, N@20: 0.4329, M@20: 0.3582\n",
      "\n",
      "test H@5: 0.4942, N@5: 0.3808, M@5: 0.3432\n",
      "test H@10: 0.6016, N@10: 0.4156, M@10: 0.3576\n",
      "test H@20: 0.7007, N@20: 0.4409, M@20: 0.3647\n",
      "\n",
      "Epoch [98/500], Train Loss: 19.9199, Elapsed: train 0.44 test 1.73 *\n",
      "valid H@5: 0.4882, N@5: 0.3750, M@5: 0.3374\n",
      "valid H@10: 0.5942, N@10: 0.4093, M@10: 0.3516\n",
      "valid H@20: 0.6869, N@20: 0.4328, M@20: 0.3581\n",
      "\n",
      "test H@5: 0.4945, N@5: 0.3828, M@5: 0.3457\n",
      "test H@10: 0.5995, N@10: 0.4168, M@10: 0.3598\n",
      "test H@20: 0.7005, N@20: 0.4427, M@20: 0.3671\n",
      "\n",
      "Epoch [99/500], Train Loss: 19.7790, Elapsed: train 0.47 test 1.81 *\n",
      "valid H@5: 0.4890, N@5: 0.3754, M@5: 0.3377\n",
      "valid H@10: 0.5913, N@10: 0.4087, M@10: 0.3516\n",
      "valid H@20: 0.6867, N@20: 0.4330, M@20: 0.3583\n",
      "\n",
      "test H@5: 0.4988, N@5: 0.3850, M@5: 0.3473\n",
      "test H@10: 0.6018, N@10: 0.4184, M@10: 0.3611\n",
      "test H@20: 0.7019, N@20: 0.4439, M@20: 0.3682\n",
      "\n",
      "Epoch [100/500], Train Loss: 19.6694, Elapsed: train 0.51 test 1.80\n",
      "valid H@5: 0.4884, N@5: 0.3752, M@5: 0.3376\n",
      "valid H@10: 0.5940, N@10: 0.4094, M@10: 0.3518\n",
      "valid H@20: 0.6877, N@20: 0.4332, M@20: 0.3584\n",
      "\n",
      "test H@5: 0.4988, N@5: 0.3853, M@5: 0.3476\n",
      "test H@10: 0.6041, N@10: 0.4193, M@10: 0.3616\n",
      "test H@20: 0.7003, N@20: 0.4439, M@20: 0.3685\n",
      "\n",
      "Epoch [101/500], Train Loss: 19.9077, Elapsed: train 0.45 test 1.77 *\n",
      "valid H@5: 0.4903, N@5: 0.3745, M@5: 0.3361\n",
      "valid H@10: 0.5947, N@10: 0.4084, M@10: 0.3502\n",
      "valid H@20: 0.6886, N@20: 0.4323, M@20: 0.3568\n",
      "\n",
      "test H@5: 0.5003, N@5: 0.3865, M@5: 0.3488\n",
      "test H@10: 0.6047, N@10: 0.4202, M@10: 0.3626\n",
      "test H@20: 0.7007, N@20: 0.4446, M@20: 0.3694\n",
      "\n",
      "Epoch [102/500], Train Loss: 19.4397, Elapsed: train 0.48 test 1.79 *\n",
      "valid H@5: 0.4919, N@5: 0.3768, M@5: 0.3387\n",
      "valid H@10: 0.5938, N@10: 0.4099, M@10: 0.3524\n",
      "valid H@20: 0.6907, N@20: 0.4345, M@20: 0.3592\n",
      "\n",
      "test H@5: 0.5026, N@5: 0.3862, M@5: 0.3476\n",
      "test H@10: 0.6039, N@10: 0.4190, M@10: 0.3611\n",
      "test H@20: 0.7042, N@20: 0.4444, M@20: 0.3682\n",
      "\n",
      "Epoch [103/500], Train Loss: 19.4897, Elapsed: train 0.45 test 1.78 *\n",
      "valid H@5: 0.4924, N@5: 0.3776, M@5: 0.3396\n",
      "valid H@10: 0.5902, N@10: 0.4095, M@10: 0.3529\n",
      "valid H@20: 0.6906, N@20: 0.4350, M@20: 0.3600\n",
      "\n",
      "test H@5: 0.5026, N@5: 0.3870, M@5: 0.3487\n",
      "test H@10: 0.6072, N@10: 0.4209, M@10: 0.3627\n",
      "test H@20: 0.7032, N@20: 0.4452, M@20: 0.3694\n",
      "\n",
      "Epoch [104/500], Train Loss: 19.4756, Elapsed: train 0.52 test 1.91 *\n",
      "valid H@5: 0.4924, N@5: 0.3779, M@5: 0.3399\n",
      "valid H@10: 0.5928, N@10: 0.4105, M@10: 0.3534\n",
      "valid H@20: 0.6923, N@20: 0.4357, M@20: 0.3604\n",
      "\n",
      "test H@5: 0.5030, N@5: 0.3866, M@5: 0.3480\n",
      "test H@10: 0.6064, N@10: 0.4201, M@10: 0.3619\n",
      "test H@20: 0.7036, N@20: 0.4447, M@20: 0.3686\n",
      "\n",
      "Epoch [105/500], Train Loss: 19.5016, Elapsed: train 0.47 test 1.73 *\n",
      "valid H@5: 0.4945, N@5: 0.3783, M@5: 0.3397\n",
      "valid H@10: 0.5919, N@10: 0.4099, M@10: 0.3528\n",
      "valid H@20: 0.6934, N@20: 0.4356, M@20: 0.3599\n",
      "\n",
      "test H@5: 0.5045, N@5: 0.3883, M@5: 0.3497\n",
      "test H@10: 0.6068, N@10: 0.4214, M@10: 0.3634\n",
      "test H@20: 0.7030, N@20: 0.4458, M@20: 0.3701\n",
      "\n",
      "Epoch [106/500], Train Loss: 19.3343, Elapsed: train 0.49 test 1.79\n",
      "valid H@5: 0.4928, N@5: 0.3789, M@5: 0.3411\n",
      "valid H@10: 0.5944, N@10: 0.4119, M@10: 0.3548\n",
      "valid H@20: 0.6915, N@20: 0.4364, M@20: 0.3615\n",
      "\n",
      "test H@5: 0.5028, N@5: 0.3883, M@5: 0.3504\n",
      "test H@10: 0.6072, N@10: 0.4221, M@10: 0.3644\n",
      "test H@20: 0.7038, N@20: 0.4466, M@20: 0.3711\n",
      "\n",
      "Epoch [107/500], Train Loss: 19.5032, Elapsed: train 0.51 test 1.81 *\n",
      "valid H@5: 0.4951, N@5: 0.3795, M@5: 0.3411\n",
      "valid H@10: 0.5961, N@10: 0.4122, M@10: 0.3547\n",
      "valid H@20: 0.6915, N@20: 0.4363, M@20: 0.3612\n",
      "\n",
      "test H@5: 0.5051, N@5: 0.3891, M@5: 0.3507\n",
      "test H@10: 0.6064, N@10: 0.4220, M@10: 0.3644\n",
      "test H@20: 0.7024, N@20: 0.4463, M@20: 0.3711\n",
      "\n",
      "Epoch [108/500], Train Loss: 19.5706, Elapsed: train 0.47 test 1.79\n",
      "valid H@5: 0.4947, N@5: 0.3791, M@5: 0.3408\n",
      "valid H@10: 0.5944, N@10: 0.4115, M@10: 0.3543\n",
      "valid H@20: 0.6923, N@20: 0.4362, M@20: 0.3610\n",
      "\n",
      "test H@5: 0.5034, N@5: 0.3884, M@5: 0.3503\n",
      "test H@10: 0.6097, N@10: 0.4229, M@10: 0.3646\n",
      "test H@20: 0.7032, N@20: 0.4465, M@20: 0.3711\n",
      "\n",
      "Epoch [109/500], Train Loss: 19.0588, Elapsed: train 0.47 test 1.76\n",
      "valid H@5: 0.4936, N@5: 0.3798, M@5: 0.3422\n",
      "valid H@10: 0.5944, N@10: 0.4126, M@10: 0.3557\n",
      "valid H@20: 0.6898, N@20: 0.4368, M@20: 0.3624\n",
      "\n",
      "test H@5: 0.5043, N@5: 0.3884, M@5: 0.3501\n",
      "test H@10: 0.6072, N@10: 0.4218, M@10: 0.3639\n",
      "test H@20: 0.7051, N@20: 0.4466, M@20: 0.3707\n",
      "\n",
      "Epoch [110/500], Train Loss: 19.3002, Elapsed: train 0.50 test 1.79\n",
      "valid H@5: 0.4905, N@5: 0.3788, M@5: 0.3418\n",
      "valid H@10: 0.5951, N@10: 0.4130, M@10: 0.3561\n",
      "valid H@20: 0.6927, N@20: 0.4376, M@20: 0.3628\n",
      "\n",
      "test H@5: 0.5034, N@5: 0.3890, M@5: 0.3511\n",
      "test H@10: 0.6057, N@10: 0.4223, M@10: 0.3650\n",
      "test H@20: 0.7053, N@20: 0.4476, M@20: 0.3720\n",
      "\n",
      "Epoch [111/500], Train Loss: 19.3709, Elapsed: train 0.46 test 1.82\n",
      "valid H@5: 0.4932, N@5: 0.3800, M@5: 0.3425\n",
      "valid H@10: 0.5947, N@10: 0.4130, M@10: 0.3562\n",
      "valid H@20: 0.6929, N@20: 0.4378, M@20: 0.3630\n",
      "\n",
      "test H@5: 0.5030, N@5: 0.3886, M@5: 0.3507\n",
      "test H@10: 0.6078, N@10: 0.4226, M@10: 0.3648\n",
      "test H@20: 0.7049, N@20: 0.4473, M@20: 0.3717\n",
      "\n",
      "Epoch [112/500], Train Loss: 19.0345, Elapsed: train 0.49 test 1.72\n",
      "valid H@5: 0.4926, N@5: 0.3810, M@5: 0.3441\n",
      "valid H@10: 0.5932, N@10: 0.4138, M@10: 0.3577\n",
      "valid H@20: 0.6875, N@20: 0.4377, M@20: 0.3643\n",
      "\n",
      "test H@5: 0.5060, N@5: 0.3892, M@5: 0.3505\n",
      "test H@10: 0.6072, N@10: 0.4221, M@10: 0.3641\n",
      "test H@20: 0.7038, N@20: 0.4466, M@20: 0.3709\n",
      "\n",
      "Epoch [113/500], Train Loss: 18.8954, Elapsed: train 0.48 test 1.79\n",
      "valid H@5: 0.4942, N@5: 0.3819, M@5: 0.3447\n",
      "valid H@10: 0.5925, N@10: 0.4139, M@10: 0.3580\n",
      "valid H@20: 0.6883, N@20: 0.4382, M@20: 0.3647\n",
      "\n",
      "test H@5: 0.5057, N@5: 0.3900, M@5: 0.3517\n",
      "test H@10: 0.6089, N@10: 0.4234, M@10: 0.3655\n",
      "test H@20: 0.7042, N@20: 0.4476, M@20: 0.3722\n",
      "\n",
      "Epoch [114/500], Train Loss: 18.8001, Elapsed: train 0.50 test 1.79\n",
      "valid H@5: 0.4951, N@5: 0.3817, M@5: 0.3442\n",
      "valid H@10: 0.5953, N@10: 0.4143, M@10: 0.3576\n",
      "valid H@20: 0.6890, N@20: 0.4379, M@20: 0.3641\n",
      "\n",
      "test H@5: 0.5068, N@5: 0.3907, M@5: 0.3523\n",
      "test H@10: 0.6082, N@10: 0.4236, M@10: 0.3659\n",
      "test H@20: 0.7034, N@20: 0.4478, M@20: 0.3726\n",
      "\n",
      "Epoch [115/500], Train Loss: 18.7584, Elapsed: train 0.49 test 1.79 *\n",
      "valid H@5: 0.4963, N@5: 0.3811, M@5: 0.3429\n",
      "valid H@10: 0.5957, N@10: 0.4133, M@10: 0.3563\n",
      "valid H@20: 0.6894, N@20: 0.4370, M@20: 0.3628\n",
      "\n",
      "test H@5: 0.5116, N@5: 0.3930, M@5: 0.3538\n",
      "test H@10: 0.6095, N@10: 0.4247, M@10: 0.3669\n",
      "test H@20: 0.7053, N@20: 0.4490, M@20: 0.3737\n",
      "\n",
      "Epoch [116/500], Train Loss: 18.6284, Elapsed: train 0.46 test 1.81 *\n",
      "valid H@5: 0.4970, N@5: 0.3827, M@5: 0.3449\n",
      "valid H@10: 0.5951, N@10: 0.4146, M@10: 0.3581\n",
      "valid H@20: 0.6884, N@20: 0.4382, M@20: 0.3646\n",
      "\n",
      "test H@5: 0.5114, N@5: 0.3920, M@5: 0.3526\n",
      "test H@10: 0.6087, N@10: 0.4237, M@10: 0.3657\n",
      "test H@20: 0.7074, N@20: 0.4487, M@20: 0.3726\n",
      "\n",
      "Epoch [117/500], Train Loss: 18.5858, Elapsed: train 0.56 test 2.22\n",
      "valid H@5: 0.4970, N@5: 0.3823, M@5: 0.3443\n",
      "valid H@10: 0.5961, N@10: 0.4145, M@10: 0.3576\n",
      "valid H@20: 0.6875, N@20: 0.4376, M@20: 0.3640\n",
      "\n",
      "test H@5: 0.5122, N@5: 0.3943, M@5: 0.3554\n",
      "test H@10: 0.6087, N@10: 0.4258, M@10: 0.3685\n",
      "test H@20: 0.7076, N@20: 0.4508, M@20: 0.3754\n",
      "\n",
      "Epoch [118/500], Train Loss: 18.8695, Elapsed: train 0.49 test 1.72 *\n",
      "valid H@5: 0.4986, N@5: 0.3833, M@5: 0.3451\n",
      "valid H@10: 0.5978, N@10: 0.4154, M@10: 0.3583\n",
      "valid H@20: 0.6898, N@20: 0.4386, M@20: 0.3647\n",
      "\n",
      "test H@5: 0.5126, N@5: 0.3936, M@5: 0.3543\n",
      "test H@10: 0.6091, N@10: 0.4250, M@10: 0.3674\n",
      "test H@20: 0.7093, N@20: 0.4503, M@20: 0.3743\n",
      "\n",
      "Epoch [119/500], Train Loss: 18.6519, Elapsed: train 0.46 test 1.77 *\n",
      "valid H@5: 0.4989, N@5: 0.3837, M@5: 0.3456\n",
      "valid H@10: 0.5953, N@10: 0.4150, M@10: 0.3585\n",
      "valid H@20: 0.6886, N@20: 0.4386, M@20: 0.3650\n",
      "\n",
      "test H@5: 0.5127, N@5: 0.3944, M@5: 0.3552\n",
      "test H@10: 0.6087, N@10: 0.4255, M@10: 0.3681\n",
      "test H@20: 0.7065, N@20: 0.4503, M@20: 0.3750\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [120/500], Train Loss: 18.7800, Elapsed: train 0.52 test 1.91 *\n",
      "valid H@5: 0.4991, N@5: 0.3823, M@5: 0.3436\n",
      "valid H@10: 0.5957, N@10: 0.4136, M@10: 0.3566\n",
      "valid H@20: 0.6911, N@20: 0.4378, M@20: 0.3633\n",
      "\n",
      "test H@5: 0.5147, N@5: 0.3955, M@5: 0.3561\n",
      "test H@10: 0.6105, N@10: 0.4265, M@10: 0.3689\n",
      "test H@20: 0.7059, N@20: 0.4507, M@20: 0.3756\n",
      "\n",
      "Epoch [121/500], Train Loss: 18.6241, Elapsed: train 0.51 test 1.81\n",
      "valid H@5: 0.4980, N@5: 0.3825, M@5: 0.3442\n",
      "valid H@10: 0.5978, N@10: 0.4149, M@10: 0.3577\n",
      "valid H@20: 0.6909, N@20: 0.4385, M@20: 0.3642\n",
      "\n",
      "test H@5: 0.5108, N@5: 0.3924, M@5: 0.3532\n",
      "test H@10: 0.6120, N@10: 0.4252, M@10: 0.3669\n",
      "test H@20: 0.7084, N@20: 0.4497, M@20: 0.3736\n",
      "\n",
      "Epoch [122/500], Train Loss: 18.6301, Elapsed: train 0.49 test 1.74 *\n",
      "valid H@5: 0.5001, N@5: 0.3829, M@5: 0.3441\n",
      "valid H@10: 0.5978, N@10: 0.4146, M@10: 0.3573\n",
      "valid H@20: 0.6898, N@20: 0.4379, M@20: 0.3637\n",
      "\n",
      "test H@5: 0.5118, N@5: 0.3922, M@5: 0.3527\n",
      "test H@10: 0.6128, N@10: 0.4251, M@10: 0.3663\n",
      "test H@20: 0.7099, N@20: 0.4496, M@20: 0.3731\n",
      "\n",
      "Epoch [123/500], Train Loss: 18.3649, Elapsed: train 0.48 test 1.72 *\n",
      "valid H@5: 0.5003, N@5: 0.3838, M@5: 0.3452\n",
      "valid H@10: 0.5978, N@10: 0.4156, M@10: 0.3584\n",
      "valid H@20: 0.6894, N@20: 0.4388, M@20: 0.3649\n",
      "\n",
      "test H@5: 0.5087, N@5: 0.3926, M@5: 0.3541\n",
      "test H@10: 0.6105, N@10: 0.4257, M@10: 0.3679\n",
      "test H@20: 0.7084, N@20: 0.4505, M@20: 0.3748\n",
      "\n",
      "Epoch [124/500], Train Loss: 18.6398, Elapsed: train 0.55 test 1.76 *\n",
      "valid H@5: 0.4968, N@5: 0.3819, M@5: 0.3437\n",
      "valid H@10: 0.5993, N@10: 0.4153, M@10: 0.3576\n",
      "valid H@20: 0.6886, N@20: 0.4379, M@20: 0.3639\n",
      "\n",
      "test H@5: 0.5076, N@5: 0.3918, M@5: 0.3534\n",
      "test H@10: 0.6133, N@10: 0.4260, M@10: 0.3676\n",
      "test H@20: 0.7101, N@20: 0.4505, M@20: 0.3743\n",
      "\n",
      "Epoch [125/500], Train Loss: 18.3795, Elapsed: train 0.54 test 1.97\n",
      "valid H@5: 0.4943, N@5: 0.3811, M@5: 0.3435\n",
      "valid H@10: 0.5974, N@10: 0.4147, M@10: 0.3575\n",
      "valid H@20: 0.6900, N@20: 0.4382, M@20: 0.3640\n",
      "\n",
      "test H@5: 0.5068, N@5: 0.3923, M@5: 0.3544\n",
      "test H@10: 0.6145, N@10: 0.4271, M@10: 0.3688\n",
      "test H@20: 0.7074, N@20: 0.4507, M@20: 0.3752\n",
      "\n",
      "Epoch [126/500], Train Loss: 17.9963, Elapsed: train 0.53 test 1.76\n",
      "valid H@5: 0.4984, N@5: 0.3822, M@5: 0.3437\n",
      "valid H@10: 0.5986, N@10: 0.4149, M@10: 0.3573\n",
      "valid H@20: 0.6923, N@20: 0.4386, M@20: 0.3638\n",
      "\n",
      "test H@5: 0.5103, N@5: 0.3922, M@5: 0.3531\n",
      "test H@10: 0.6151, N@10: 0.4261, M@10: 0.3671\n",
      "test H@20: 0.7074, N@20: 0.4495, M@20: 0.3736\n",
      "\n",
      "Epoch [127/500], Train Loss: 18.2041, Elapsed: train 0.52 test 1.76\n",
      "valid H@5: 0.4966, N@5: 0.3818, M@5: 0.3436\n",
      "valid H@10: 0.5984, N@10: 0.4149, M@10: 0.3575\n",
      "valid H@20: 0.6932, N@20: 0.4389, M@20: 0.3640\n",
      "\n",
      "test H@5: 0.5093, N@5: 0.3919, M@5: 0.3530\n",
      "test H@10: 0.6143, N@10: 0.4260, M@10: 0.3672\n",
      "test H@20: 0.7066, N@20: 0.4495, M@20: 0.3737\n",
      "\n",
      "Epoch [128/500], Train Loss: 18.3411, Elapsed: train 0.47 test 1.88 *\n",
      "valid H@5: 0.4972, N@5: 0.3827, M@5: 0.3448\n",
      "valid H@10: 0.6005, N@10: 0.4163, M@10: 0.3587\n",
      "valid H@20: 0.6930, N@20: 0.4397, M@20: 0.3651\n",
      "\n",
      "test H@5: 0.5103, N@5: 0.3923, M@5: 0.3532\n",
      "test H@10: 0.6143, N@10: 0.4261, M@10: 0.3673\n",
      "test H@20: 0.7080, N@20: 0.4498, M@20: 0.3738\n",
      "\n",
      "Epoch [129/500], Train Loss: 18.2390, Elapsed: train 0.43 test 1.96\n",
      "valid H@5: 0.4993, N@5: 0.3835, M@5: 0.3451\n",
      "valid H@10: 0.6005, N@10: 0.4164, M@10: 0.3588\n",
      "valid H@20: 0.6929, N@20: 0.4398, M@20: 0.3652\n",
      "\n",
      "test H@5: 0.5103, N@5: 0.3932, M@5: 0.3545\n",
      "test H@10: 0.6154, N@10: 0.4273, M@10: 0.3686\n",
      "test H@20: 0.7086, N@20: 0.4509, M@20: 0.3752\n",
      "\n",
      "Epoch [130/500], Train Loss: 18.0856, Elapsed: train 0.57 test 1.85 *\n",
      "valid H@5: 0.4991, N@5: 0.3850, M@5: 0.3472\n",
      "valid H@10: 0.6011, N@10: 0.4181, M@10: 0.3609\n",
      "valid H@20: 0.6963, N@20: 0.4420, M@20: 0.3674\n",
      "\n",
      "test H@5: 0.5106, N@5: 0.3928, M@5: 0.3538\n",
      "test H@10: 0.6202, N@10: 0.4282, M@10: 0.3684\n",
      "test H@20: 0.7095, N@20: 0.4507, M@20: 0.3745\n",
      "\n",
      "Epoch [131/500], Train Loss: 18.1238, Elapsed: train 0.51 test 1.94\n",
      "valid H@5: 0.4972, N@5: 0.3838, M@5: 0.3461\n",
      "valid H@10: 0.5999, N@10: 0.4172, M@10: 0.3601\n",
      "valid H@20: 0.6961, N@20: 0.4415, M@20: 0.3667\n",
      "\n",
      "test H@5: 0.5091, N@5: 0.3920, M@5: 0.3531\n",
      "test H@10: 0.6199, N@10: 0.4279, M@10: 0.3680\n",
      "test H@20: 0.7091, N@20: 0.4505, M@20: 0.3742\n",
      "\n",
      "Epoch [132/500], Train Loss: 18.1426, Elapsed: train 0.46 test 1.80 *\n",
      "valid H@5: 0.5003, N@5: 0.3851, M@5: 0.3469\n",
      "valid H@10: 0.6024, N@10: 0.4182, M@10: 0.3606\n",
      "valid H@20: 0.6973, N@20: 0.4420, M@20: 0.3670\n",
      "\n",
      "test H@5: 0.5108, N@5: 0.3938, M@5: 0.3551\n",
      "test H@10: 0.6189, N@10: 0.4288, M@10: 0.3695\n",
      "test H@20: 0.7109, N@20: 0.4520, M@20: 0.3759\n",
      "\n",
      "Epoch [133/500], Train Loss: 17.9314, Elapsed: train 0.47 test 1.79\n",
      "valid H@5: 0.4988, N@5: 0.3838, M@5: 0.3457\n",
      "valid H@10: 0.6020, N@10: 0.4173, M@10: 0.3595\n",
      "valid H@20: 0.6963, N@20: 0.4411, M@20: 0.3660\n",
      "\n",
      "test H@5: 0.5089, N@5: 0.3924, M@5: 0.3538\n",
      "test H@10: 0.6200, N@10: 0.4283, M@10: 0.3686\n",
      "test H@20: 0.7093, N@20: 0.4510, M@20: 0.3749\n",
      "\n",
      "Epoch [134/500], Train Loss: 18.4328, Elapsed: train 0.55 test 1.78\n",
      "valid H@5: 0.4968, N@5: 0.3840, M@5: 0.3464\n",
      "valid H@10: 0.6007, N@10: 0.4179, M@10: 0.3607\n",
      "valid H@20: 0.6969, N@20: 0.4423, M@20: 0.3674\n",
      "\n",
      "test H@5: 0.5114, N@5: 0.3944, M@5: 0.3557\n",
      "test H@10: 0.6179, N@10: 0.4289, M@10: 0.3699\n",
      "test H@20: 0.7093, N@20: 0.4520, M@20: 0.3763\n",
      "\n",
      "Epoch [135/500], Train Loss: 18.1069, Elapsed: train 0.48 test 1.76\n",
      "valid H@5: 0.4972, N@5: 0.3841, M@5: 0.3466\n",
      "valid H@10: 0.6003, N@10: 0.4179, M@10: 0.3608\n",
      "valid H@20: 0.6965, N@20: 0.4423, M@20: 0.3675\n",
      "\n",
      "test H@5: 0.5131, N@5: 0.3954, M@5: 0.3564\n",
      "test H@10: 0.6145, N@10: 0.4282, M@10: 0.3700\n",
      "test H@20: 0.7112, N@20: 0.4528, M@20: 0.3768\n",
      "\n",
      "Epoch [136/500], Train Loss: 18.1685, Elapsed: train 0.52 test 1.81\n",
      "valid H@5: 0.4978, N@5: 0.3840, M@5: 0.3462\n",
      "valid H@10: 0.6011, N@10: 0.4177, M@10: 0.3603\n",
      "valid H@20: 0.6959, N@20: 0.4417, M@20: 0.3669\n",
      "\n",
      "test H@5: 0.5103, N@5: 0.3944, M@5: 0.3560\n",
      "test H@10: 0.6176, N@10: 0.4291, M@10: 0.3703\n",
      "test H@20: 0.7091, N@20: 0.4523, M@20: 0.3767\n",
      "\n",
      "Epoch [137/500], Train Loss: 18.0347, Elapsed: train 0.50 test 1.83 *\n",
      "valid H@5: 0.4986, N@5: 0.3832, M@5: 0.3450\n",
      "valid H@10: 0.6003, N@10: 0.4164, M@10: 0.3588\n",
      "valid H@20: 0.6980, N@20: 0.4410, M@20: 0.3655\n",
      "\n",
      "test H@5: 0.5122, N@5: 0.3952, M@5: 0.3564\n",
      "test H@10: 0.6162, N@10: 0.4288, M@10: 0.3703\n",
      "test H@20: 0.7097, N@20: 0.4525, M@20: 0.3768\n",
      "\n",
      "Epoch [138/500], Train Loss: 17.9334, Elapsed: train 0.46 test 1.80 *\n",
      "valid H@5: 0.5020, N@5: 0.3853, M@5: 0.3465\n",
      "valid H@10: 0.6007, N@10: 0.4174, M@10: 0.3599\n",
      "valid H@20: 0.6982, N@20: 0.4420, M@20: 0.3666\n",
      "\n",
      "test H@5: 0.5091, N@5: 0.3935, M@5: 0.3552\n",
      "test H@10: 0.6153, N@10: 0.4281, M@10: 0.3696\n",
      "test H@20: 0.7086, N@20: 0.4517, M@20: 0.3762\n",
      "\n",
      "Epoch [139/500], Train Loss: 17.7257, Elapsed: train 0.47 test 1.80\n",
      "valid H@5: 0.4982, N@5: 0.3844, M@5: 0.3466\n",
      "valid H@10: 0.6016, N@10: 0.4182, M@10: 0.3607\n",
      "valid H@20: 0.6975, N@20: 0.4424, M@20: 0.3673\n",
      "\n",
      "test H@5: 0.5110, N@5: 0.3943, M@5: 0.3556\n",
      "test H@10: 0.6183, N@10: 0.4291, M@10: 0.3700\n",
      "test H@20: 0.7120, N@20: 0.4528, M@20: 0.3765\n",
      "\n",
      "Epoch [140/500], Train Loss: 17.8001, Elapsed: train 0.56 test 1.89\n",
      "valid H@5: 0.5009, N@5: 0.3854, M@5: 0.3470\n",
      "valid H@10: 0.6016, N@10: 0.4182, M@10: 0.3607\n",
      "valid H@20: 0.6971, N@20: 0.4424, M@20: 0.3674\n",
      "\n",
      "test H@5: 0.5129, N@5: 0.3950, M@5: 0.3559\n",
      "test H@10: 0.6204, N@10: 0.4298, M@10: 0.3703\n",
      "test H@20: 0.7112, N@20: 0.4527, M@20: 0.3766\n",
      "\n",
      "Epoch [141/500], Train Loss: 18.0040, Elapsed: train 0.53 test 1.76 *\n",
      "valid H@5: 0.5009, N@5: 0.3859, M@5: 0.3477\n",
      "valid H@10: 0.6028, N@10: 0.4190, M@10: 0.3615\n",
      "valid H@20: 0.6982, N@20: 0.4432, M@20: 0.3681\n",
      "\n",
      "test H@5: 0.5148, N@5: 0.3964, M@5: 0.3572\n",
      "test H@10: 0.6187, N@10: 0.4300, M@10: 0.3711\n",
      "test H@20: 0.7107, N@20: 0.4533, M@20: 0.3775\n",
      "\n",
      "Epoch [142/500], Train Loss: 18.0834, Elapsed: train 0.45 test 1.76 *\n",
      "valid H@5: 0.4993, N@5: 0.3858, M@5: 0.3481\n",
      "valid H@10: 0.6034, N@10: 0.4196, M@10: 0.3621\n",
      "valid H@20: 0.6996, N@20: 0.4439, M@20: 0.3688\n",
      "\n",
      "test H@5: 0.5135, N@5: 0.3978, M@5: 0.3594\n",
      "test H@10: 0.6189, N@10: 0.4320, M@10: 0.3736\n",
      "test H@20: 0.7122, N@20: 0.4556, M@20: 0.3801\n",
      "\n",
      "Epoch [143/500], Train Loss: 17.8703, Elapsed: train 0.51 test 1.82\n",
      "valid H@5: 0.5012, N@5: 0.3870, M@5: 0.3490\n",
      "valid H@10: 0.6020, N@10: 0.4199, M@10: 0.3627\n",
      "valid H@20: 0.6994, N@20: 0.4444, M@20: 0.3694\n",
      "\n",
      "test H@5: 0.5156, N@5: 0.3992, M@5: 0.3606\n",
      "test H@10: 0.6210, N@10: 0.4333, M@10: 0.3747\n",
      "test H@20: 0.7112, N@20: 0.4561, M@20: 0.3810\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [144/500], Train Loss: 17.6609, Elapsed: train 0.54 test 1.76 *\n",
      "valid H@5: 0.5026, N@5: 0.3871, M@5: 0.3488\n",
      "valid H@10: 0.6022, N@10: 0.4196, M@10: 0.3624\n",
      "valid H@20: 0.6965, N@20: 0.4434, M@20: 0.3689\n",
      "\n",
      "test H@5: 0.5160, N@5: 0.3988, M@5: 0.3599\n",
      "test H@10: 0.6189, N@10: 0.4322, M@10: 0.3738\n",
      "test H@20: 0.7124, N@20: 0.4558, M@20: 0.3803\n",
      "\n",
      "Epoch [145/500], Train Loss: 17.5927, Elapsed: train 0.48 test 1.81 *\n",
      "valid H@5: 0.5003, N@5: 0.3855, M@5: 0.3474\n",
      "valid H@10: 0.6055, N@10: 0.4197, M@10: 0.3616\n",
      "valid H@20: 0.6953, N@20: 0.4424, M@20: 0.3678\n",
      "\n",
      "test H@5: 0.5150, N@5: 0.3994, M@5: 0.3610\n",
      "test H@10: 0.6214, N@10: 0.4340, M@10: 0.3754\n",
      "test H@20: 0.7111, N@20: 0.4566, M@20: 0.3816\n",
      "\n",
      "Epoch [146/500], Train Loss: 17.6923, Elapsed: train 0.44 test 1.79\n",
      "valid H@5: 0.5026, N@5: 0.3862, M@5: 0.3476\n",
      "valid H@10: 0.6047, N@10: 0.4193, M@10: 0.3613\n",
      "valid H@20: 0.6959, N@20: 0.4423, M@20: 0.3676\n",
      "\n",
      "test H@5: 0.5160, N@5: 0.4008, M@5: 0.3626\n",
      "test H@10: 0.6204, N@10: 0.4347, M@10: 0.3768\n",
      "test H@20: 0.7122, N@20: 0.4579, M@20: 0.3831\n",
      "\n",
      "Epoch [147/500], Train Loss: 17.7858, Elapsed: train 0.53 test 1.79\n",
      "valid H@5: 0.5018, N@5: 0.3870, M@5: 0.3489\n",
      "valid H@10: 0.6036, N@10: 0.4201, M@10: 0.3627\n",
      "valid H@20: 0.6961, N@20: 0.4434, M@20: 0.3691\n",
      "\n",
      "test H@5: 0.5139, N@5: 0.3985, M@5: 0.3603\n",
      "test H@10: 0.6185, N@10: 0.4325, M@10: 0.3744\n",
      "test H@20: 0.7114, N@20: 0.4561, M@20: 0.3809\n",
      "\n",
      "Epoch [148/500], Train Loss: 17.6109, Elapsed: train 0.47 test 1.81\n",
      "valid H@5: 0.5018, N@5: 0.3869, M@5: 0.3487\n",
      "valid H@10: 0.6038, N@10: 0.4200, M@10: 0.3625\n",
      "valid H@20: 0.6957, N@20: 0.4433, M@20: 0.3689\n",
      "\n",
      "test H@5: 0.5189, N@5: 0.4014, M@5: 0.3625\n",
      "test H@10: 0.6195, N@10: 0.4340, M@10: 0.3760\n",
      "test H@20: 0.7122, N@20: 0.4574, M@20: 0.3825\n",
      "\n",
      "Epoch [149/500], Train Loss: 17.6161, Elapsed: train 0.49 test 1.80\n",
      "valid H@5: 0.4991, N@5: 0.3865, M@5: 0.3490\n",
      "valid H@10: 0.6043, N@10: 0.4205, M@10: 0.3631\n",
      "valid H@20: 0.6953, N@20: 0.4435, M@20: 0.3694\n",
      "\n",
      "test H@5: 0.5171, N@5: 0.3998, M@5: 0.3610\n",
      "test H@10: 0.6185, N@10: 0.4328, M@10: 0.3747\n",
      "test H@20: 0.7139, N@20: 0.4570, M@20: 0.3814\n",
      "\n",
      "Epoch [150/500], Train Loss: 17.5651, Elapsed: train 0.50 test 1.78\n",
      "valid H@5: 0.5007, N@5: 0.3872, M@5: 0.3494\n",
      "valid H@10: 0.6041, N@10: 0.4208, M@10: 0.3634\n",
      "valid H@20: 0.6946, N@20: 0.4437, M@20: 0.3697\n",
      "\n",
      "test H@5: 0.5170, N@5: 0.3993, M@5: 0.3603\n",
      "test H@10: 0.6202, N@10: 0.4327, M@10: 0.3742\n",
      "test H@20: 0.7157, N@20: 0.4568, M@20: 0.3808\n",
      "\n",
      "Epoch [151/500], Train Loss: 17.7663, Elapsed: train 0.48 test 1.83 *\n",
      "valid H@5: 0.4995, N@5: 0.3867, M@5: 0.3492\n",
      "valid H@10: 0.6057, N@10: 0.4212, M@10: 0.3636\n",
      "valid H@20: 0.6938, N@20: 0.4436, M@20: 0.3697\n",
      "\n",
      "test H@5: 0.5177, N@5: 0.3992, M@5: 0.3599\n",
      "test H@10: 0.6212, N@10: 0.4327, M@10: 0.3738\n",
      "test H@20: 0.7160, N@20: 0.4567, M@20: 0.3804\n",
      "\n",
      "Epoch [152/500], Train Loss: 17.6246, Elapsed: train 0.50 test 1.75 *\n",
      "valid H@5: 0.5005, N@5: 0.3873, M@5: 0.3496\n",
      "valid H@10: 0.6061, N@10: 0.4215, M@10: 0.3638\n",
      "valid H@20: 0.6927, N@20: 0.4435, M@20: 0.3699\n",
      "\n",
      "test H@5: 0.5158, N@5: 0.3984, M@5: 0.3595\n",
      "test H@10: 0.6179, N@10: 0.4316, M@10: 0.3733\n",
      "test H@20: 0.7170, N@20: 0.4567, M@20: 0.3802\n",
      "\n",
      "Epoch [153/500], Train Loss: 17.1205, Elapsed: train 0.49 test 1.74 *\n",
      "valid H@5: 0.5035, N@5: 0.3878, M@5: 0.3493\n",
      "valid H@10: 0.6057, N@10: 0.4209, M@10: 0.3631\n",
      "valid H@20: 0.6959, N@20: 0.4437, M@20: 0.3693\n",
      "\n",
      "test H@5: 0.5185, N@5: 0.3996, M@5: 0.3603\n",
      "test H@10: 0.6187, N@10: 0.4320, M@10: 0.3737\n",
      "test H@20: 0.7160, N@20: 0.4566, M@20: 0.3804\n",
      "\n",
      "Epoch [154/500], Train Loss: 17.3639, Elapsed: train 0.52 test 1.84 *\n",
      "valid H@5: 0.5057, N@5: 0.3889, M@5: 0.3501\n",
      "valid H@10: 0.6053, N@10: 0.4212, M@10: 0.3635\n",
      "valid H@20: 0.6934, N@20: 0.4435, M@20: 0.3696\n",
      "\n",
      "test H@5: 0.5168, N@5: 0.3990, M@5: 0.3600\n",
      "test H@10: 0.6204, N@10: 0.4326, M@10: 0.3739\n",
      "test H@20: 0.7137, N@20: 0.4562, M@20: 0.3804\n",
      "\n",
      "Epoch [155/500], Train Loss: 17.8032, Elapsed: train 0.46 test 1.90\n",
      "valid H@5: 0.5055, N@5: 0.3896, M@5: 0.3511\n",
      "valid H@10: 0.6053, N@10: 0.4220, M@10: 0.3646\n",
      "valid H@20: 0.6961, N@20: 0.4449, M@20: 0.3709\n",
      "\n",
      "test H@5: 0.5162, N@5: 0.3988, M@5: 0.3599\n",
      "test H@10: 0.6235, N@10: 0.4334, M@10: 0.3742\n",
      "test H@20: 0.7141, N@20: 0.4564, M@20: 0.3805\n",
      "\n",
      "Epoch [156/500], Train Loss: 17.4310, Elapsed: train 0.49 test 1.86\n",
      "valid H@5: 0.5053, N@5: 0.3886, M@5: 0.3499\n",
      "valid H@10: 0.6036, N@10: 0.4205, M@10: 0.3632\n",
      "valid H@20: 0.6957, N@20: 0.4439, M@20: 0.3696\n",
      "\n",
      "test H@5: 0.5164, N@5: 0.3992, M@5: 0.3605\n",
      "test H@10: 0.6233, N@10: 0.4338, M@10: 0.3747\n",
      "test H@20: 0.7118, N@20: 0.4562, M@20: 0.3809\n",
      "\n",
      "Epoch [157/500], Train Loss: 17.1918, Elapsed: train 0.54 test 1.89 *\n",
      "valid H@5: 0.5053, N@5: 0.3878, M@5: 0.3489\n",
      "valid H@10: 0.6064, N@10: 0.4206, M@10: 0.3625\n",
      "valid H@20: 0.6988, N@20: 0.4439, M@20: 0.3688\n",
      "\n",
      "test H@5: 0.5154, N@5: 0.3997, M@5: 0.3613\n",
      "test H@10: 0.6235, N@10: 0.4346, M@10: 0.3757\n",
      "test H@20: 0.7137, N@20: 0.4575, M@20: 0.3821\n",
      "\n",
      "Epoch [158/500], Train Loss: 17.4139, Elapsed: train 0.46 test 1.74 *\n",
      "valid H@5: 0.5064, N@5: 0.3898, M@5: 0.3511\n",
      "valid H@10: 0.6070, N@10: 0.4223, M@10: 0.3645\n",
      "valid H@20: 0.6973, N@20: 0.4450, M@20: 0.3707\n",
      "\n",
      "test H@5: 0.5131, N@5: 0.3995, M@5: 0.3618\n",
      "test H@10: 0.6210, N@10: 0.4345, M@10: 0.3763\n",
      "test H@20: 0.7122, N@20: 0.4577, M@20: 0.3828\n",
      "\n",
      "Epoch [159/500], Train Loss: 17.1053, Elapsed: train 0.47 test 1.78\n",
      "valid H@5: 0.5039, N@5: 0.3893, M@5: 0.3512\n",
      "valid H@10: 0.6066, N@10: 0.4226, M@10: 0.3650\n",
      "valid H@20: 0.6967, N@20: 0.4453, M@20: 0.3713\n",
      "\n",
      "test H@5: 0.5122, N@5: 0.3995, M@5: 0.3621\n",
      "test H@10: 0.6174, N@10: 0.4336, M@10: 0.3763\n",
      "test H@20: 0.7141, N@20: 0.4583, M@20: 0.3832\n",
      "\n",
      "Epoch [160/500], Train Loss: 17.2107, Elapsed: train 0.51 test 1.78\n",
      "valid H@5: 0.5037, N@5: 0.3884, M@5: 0.3501\n",
      "valid H@10: 0.6068, N@10: 0.4218, M@10: 0.3640\n",
      "valid H@20: 0.6975, N@20: 0.4447, M@20: 0.3702\n",
      "\n",
      "test H@5: 0.5131, N@5: 0.4002, M@5: 0.3628\n",
      "test H@10: 0.6189, N@10: 0.4346, M@10: 0.3771\n",
      "test H@20: 0.7162, N@20: 0.4593, M@20: 0.3839\n",
      "\n",
      "Epoch [161/500], Train Loss: 17.2142, Elapsed: train 0.42 test 1.71\n",
      "valid H@5: 0.5057, N@5: 0.3887, M@5: 0.3500\n",
      "valid H@10: 0.6068, N@10: 0.4215, M@10: 0.3635\n",
      "valid H@20: 0.6971, N@20: 0.4442, M@20: 0.3697\n",
      "\n",
      "test H@5: 0.5137, N@5: 0.3996, M@5: 0.3618\n",
      "test H@10: 0.6197, N@10: 0.4340, M@10: 0.3761\n",
      "test H@20: 0.7134, N@20: 0.4578, M@20: 0.3826\n",
      "\n",
      "Epoch [162/500], Train Loss: 17.2825, Elapsed: train 0.47 test 1.85\n",
      "valid H@5: 0.5043, N@5: 0.3884, M@5: 0.3500\n",
      "valid H@10: 0.6043, N@10: 0.4209, M@10: 0.3635\n",
      "valid H@20: 0.6986, N@20: 0.4447, M@20: 0.3700\n",
      "\n",
      "test H@5: 0.5137, N@5: 0.3991, M@5: 0.3612\n",
      "test H@10: 0.6187, N@10: 0.4332, M@10: 0.3753\n",
      "test H@20: 0.7141, N@20: 0.4574, M@20: 0.3820\n",
      "\n",
      "Epoch [163/500], Train Loss: 17.4148, Elapsed: train 0.52 test 1.79\n",
      "valid H@5: 0.5051, N@5: 0.3888, M@5: 0.3503\n",
      "valid H@10: 0.6028, N@10: 0.4205, M@10: 0.3634\n",
      "valid H@20: 0.6988, N@20: 0.4448, M@20: 0.3701\n",
      "\n",
      "test H@5: 0.5154, N@5: 0.3993, M@5: 0.3608\n",
      "test H@10: 0.6195, N@10: 0.4331, M@10: 0.3749\n",
      "test H@20: 0.7141, N@20: 0.4571, M@20: 0.3815\n",
      "\n",
      "Epoch [164/500], Train Loss: 17.3466, Elapsed: train 0.54 test 1.79\n",
      "valid H@5: 0.5039, N@5: 0.3872, M@5: 0.3485\n",
      "valid H@10: 0.6036, N@10: 0.4197, M@10: 0.3620\n",
      "valid H@20: 0.6992, N@20: 0.4438, M@20: 0.3687\n",
      "\n",
      "test H@5: 0.5150, N@5: 0.3985, M@5: 0.3599\n",
      "test H@10: 0.6202, N@10: 0.4326, M@10: 0.3740\n",
      "test H@20: 0.7147, N@20: 0.4566, M@20: 0.3807\n",
      "\n",
      "Epoch [165/500], Train Loss: 17.1288, Elapsed: train 0.46 test 1.75 *\n",
      "valid H@5: 0.5058, N@5: 0.3883, M@5: 0.3493\n",
      "valid H@10: 0.6041, N@10: 0.4202, M@10: 0.3626\n",
      "valid H@20: 0.7001, N@20: 0.4444, M@20: 0.3692\n",
      "\n",
      "test H@5: 0.5148, N@5: 0.3998, M@5: 0.3617\n",
      "test H@10: 0.6187, N@10: 0.4335, M@10: 0.3757\n",
      "test H@20: 0.7153, N@20: 0.4580, M@20: 0.3825\n",
      "\n",
      "Epoch [166/500], Train Loss: 17.1572, Elapsed: train 0.48 test 1.98 *\n",
      "valid H@5: 0.5055, N@5: 0.3888, M@5: 0.3501\n",
      "valid H@10: 0.6041, N@10: 0.4209, M@10: 0.3634\n",
      "valid H@20: 0.7003, N@20: 0.4451, M@20: 0.3701\n",
      "\n",
      "test H@5: 0.5127, N@5: 0.3992, M@5: 0.3615\n",
      "test H@10: 0.6187, N@10: 0.4336, M@10: 0.3758\n",
      "test H@20: 0.7153, N@20: 0.4582, M@20: 0.3826\n",
      "\n",
      "Epoch [167/500], Train Loss: 17.0095, Elapsed: train 0.56 test 1.85 *\n",
      "valid H@5: 0.5068, N@5: 0.3889, M@5: 0.3498\n",
      "valid H@10: 0.6068, N@10: 0.4213, M@10: 0.3632\n",
      "valid H@20: 0.6992, N@20: 0.4446, M@20: 0.3695\n",
      "\n",
      "test H@5: 0.5168, N@5: 0.4005, M@5: 0.3620\n",
      "test H@10: 0.6187, N@10: 0.4335, M@10: 0.3756\n",
      "test H@20: 0.7155, N@20: 0.4581, M@20: 0.3824\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [168/500], Train Loss: 17.2043, Elapsed: train 0.46 test 1.78\n",
      "valid H@5: 0.5062, N@5: 0.3891, M@5: 0.3502\n",
      "valid H@10: 0.6068, N@10: 0.4217, M@10: 0.3637\n",
      "valid H@20: 0.6988, N@20: 0.4450, M@20: 0.3701\n",
      "\n",
      "test H@5: 0.5185, N@5: 0.4012, M@5: 0.3623\n",
      "test H@10: 0.6189, N@10: 0.4336, M@10: 0.3757\n",
      "test H@20: 0.7176, N@20: 0.4587, M@20: 0.3826\n",
      "\n",
      "Epoch [169/500], Train Loss: 17.2313, Elapsed: train 0.44 test 1.73\n",
      "valid H@5: 0.5043, N@5: 0.3874, M@5: 0.3486\n",
      "valid H@10: 0.6049, N@10: 0.4200, M@10: 0.3621\n",
      "valid H@20: 0.6984, N@20: 0.4437, M@20: 0.3687\n",
      "\n",
      "test H@5: 0.5166, N@5: 0.4011, M@5: 0.3628\n",
      "test H@10: 0.6197, N@10: 0.4345, M@10: 0.3766\n",
      "test H@20: 0.7139, N@20: 0.4585, M@20: 0.3833\n",
      "\n",
      "Epoch [170/500], Train Loss: 17.2694, Elapsed: train 0.54 test 1.82 *\n",
      "valid H@5: 0.5045, N@5: 0.3881, M@5: 0.3495\n",
      "valid H@10: 0.6041, N@10: 0.4204, M@10: 0.3629\n",
      "valid H@20: 0.7011, N@20: 0.4450, M@20: 0.3696\n",
      "\n",
      "test H@5: 0.5156, N@5: 0.4000, M@5: 0.3616\n",
      "test H@10: 0.6193, N@10: 0.4335, M@10: 0.3755\n",
      "test H@20: 0.7141, N@20: 0.4576, M@20: 0.3821\n",
      "\n",
      "Epoch [171/500], Train Loss: 16.8698, Elapsed: train 0.46 test 1.73\n",
      "valid H@5: 0.5062, N@5: 0.3881, M@5: 0.3489\n",
      "valid H@10: 0.6061, N@10: 0.4204, M@10: 0.3623\n",
      "valid H@20: 0.6980, N@20: 0.4437, M@20: 0.3687\n",
      "\n",
      "test H@5: 0.5166, N@5: 0.4017, M@5: 0.3637\n",
      "test H@10: 0.6223, N@10: 0.4359, M@10: 0.3777\n",
      "test H@20: 0.7139, N@20: 0.4591, M@20: 0.3841\n",
      "\n",
      "Epoch [172/500], Train Loss: 17.1753, Elapsed: train 0.48 test 1.77\n",
      "valid H@5: 0.5045, N@5: 0.3882, M@5: 0.3496\n",
      "valid H@10: 0.6064, N@10: 0.4213, M@10: 0.3633\n",
      "valid H@20: 0.6988, N@20: 0.4446, M@20: 0.3697\n",
      "\n",
      "test H@5: 0.5171, N@5: 0.4010, M@5: 0.3624\n",
      "test H@10: 0.6193, N@10: 0.4342, M@10: 0.3762\n",
      "test H@20: 0.7141, N@20: 0.4582, M@20: 0.3829\n",
      "\n",
      "Epoch [173/500], Train Loss: 17.2150, Elapsed: train 0.46 test 1.71 *\n",
      "valid H@5: 0.5049, N@5: 0.3885, M@5: 0.3498\n",
      "valid H@10: 0.6078, N@10: 0.4218, M@10: 0.3636\n",
      "valid H@20: 0.7005, N@20: 0.4452, M@20: 0.3700\n",
      "\n",
      "test H@5: 0.5173, N@5: 0.4004, M@5: 0.3617\n",
      "test H@10: 0.6223, N@10: 0.4344, M@10: 0.3757\n",
      "test H@20: 0.7134, N@20: 0.4575, M@20: 0.3821\n",
      "\n",
      "Epoch [174/500], Train Loss: 17.2451, Elapsed: train 0.51 test 1.73 *\n",
      "valid H@5: 0.5035, N@5: 0.3883, M@5: 0.3499\n",
      "valid H@10: 0.6080, N@10: 0.4221, M@10: 0.3640\n",
      "valid H@20: 0.7011, N@20: 0.4457, M@20: 0.3704\n",
      "\n",
      "test H@5: 0.5183, N@5: 0.4009, M@5: 0.3621\n",
      "test H@10: 0.6233, N@10: 0.4349, M@10: 0.3761\n",
      "test H@20: 0.7139, N@20: 0.4579, M@20: 0.3824\n",
      "\n",
      "Epoch [175/500], Train Loss: 17.1459, Elapsed: train 0.48 test 1.75 *\n",
      "valid H@5: 0.5062, N@5: 0.3901, M@5: 0.3515\n",
      "valid H@10: 0.6082, N@10: 0.4231, M@10: 0.3651\n",
      "valid H@20: 0.7005, N@20: 0.4464, M@20: 0.3715\n",
      "\n",
      "test H@5: 0.5173, N@5: 0.3997, M@5: 0.3606\n",
      "test H@10: 0.6210, N@10: 0.4333, M@10: 0.3746\n",
      "test H@20: 0.7126, N@20: 0.4566, M@20: 0.3810\n",
      "\n",
      "Epoch [176/500], Train Loss: 17.0635, Elapsed: train 0.46 test 1.81 *\n",
      "valid H@5: 0.5078, N@5: 0.3910, M@5: 0.3523\n",
      "valid H@10: 0.6080, N@10: 0.4236, M@10: 0.3658\n",
      "valid H@20: 0.7007, N@20: 0.4470, M@20: 0.3722\n",
      "\n",
      "test H@5: 0.5150, N@5: 0.3996, M@5: 0.3613\n",
      "test H@10: 0.6200, N@10: 0.4339, M@10: 0.3756\n",
      "test H@20: 0.7134, N@20: 0.4575, M@20: 0.3822\n",
      "\n",
      "Epoch [177/500], Train Loss: 17.0929, Elapsed: train 0.55 test 1.85 *\n",
      "valid H@5: 0.5083, N@5: 0.3924, M@5: 0.3539\n",
      "valid H@10: 0.6085, N@10: 0.4248, M@10: 0.3673\n",
      "valid H@20: 0.7013, N@20: 0.4482, M@20: 0.3737\n",
      "\n",
      "test H@5: 0.5189, N@5: 0.4027, M@5: 0.3642\n",
      "test H@10: 0.6204, N@10: 0.4357, M@10: 0.3779\n",
      "test H@20: 0.7122, N@20: 0.4590, M@20: 0.3843\n",
      "\n",
      "Epoch [178/500], Train Loss: 17.2017, Elapsed: train 0.46 test 1.86 *\n",
      "valid H@5: 0.5093, N@5: 0.3918, M@5: 0.3529\n",
      "valid H@10: 0.6080, N@10: 0.4237, M@10: 0.3660\n",
      "valid H@20: 0.7007, N@20: 0.4472, M@20: 0.3725\n",
      "\n",
      "test H@5: 0.5191, N@5: 0.4026, M@5: 0.3640\n",
      "test H@10: 0.6214, N@10: 0.4358, M@10: 0.3778\n",
      "test H@20: 0.7132, N@20: 0.4591, M@20: 0.3842\n",
      "\n",
      "Epoch [179/500], Train Loss: 17.1391, Elapsed: train 0.49 test 1.79 *\n",
      "valid H@5: 0.5097, N@5: 0.3913, M@5: 0.3520\n",
      "valid H@10: 0.6085, N@10: 0.4233, M@10: 0.3653\n",
      "valid H@20: 0.7003, N@20: 0.4465, M@20: 0.3716\n",
      "\n",
      "test H@5: 0.5200, N@5: 0.4028, M@5: 0.3639\n",
      "test H@10: 0.6206, N@10: 0.4354, M@10: 0.3775\n",
      "test H@20: 0.7128, N@20: 0.4589, M@20: 0.3840\n",
      "\n",
      "Epoch [180/500], Train Loss: 16.9733, Elapsed: train 0.53 test 1.79\n",
      "valid H@5: 0.5087, N@5: 0.3914, M@5: 0.3525\n",
      "valid H@10: 0.6084, N@10: 0.4236, M@10: 0.3658\n",
      "valid H@20: 0.6988, N@20: 0.4465, M@20: 0.3721\n",
      "\n",
      "test H@5: 0.5164, N@5: 0.4008, M@5: 0.3624\n",
      "test H@10: 0.6199, N@10: 0.4345, M@10: 0.3765\n",
      "test H@20: 0.7135, N@20: 0.4584, M@20: 0.3832\n",
      "\n",
      "Epoch [181/500], Train Loss: 17.2522, Elapsed: train 0.47 test 1.76 *\n",
      "valid H@5: 0.5068, N@5: 0.3909, M@5: 0.3525\n",
      "valid H@10: 0.6093, N@10: 0.4241, M@10: 0.3662\n",
      "valid H@20: 0.7003, N@20: 0.4471, M@20: 0.3725\n",
      "\n",
      "test H@5: 0.5189, N@5: 0.4031, M@5: 0.3647\n",
      "test H@10: 0.6233, N@10: 0.4370, M@10: 0.3788\n",
      "test H@20: 0.7147, N@20: 0.4602, M@20: 0.3852\n",
      "\n",
      "Epoch [182/500], Train Loss: 16.9171, Elapsed: train 0.49 test 1.78\n",
      "valid H@5: 0.5083, N@5: 0.3919, M@5: 0.3533\n",
      "valid H@10: 0.6068, N@10: 0.4240, M@10: 0.3666\n",
      "valid H@20: 0.7001, N@20: 0.4477, M@20: 0.3732\n",
      "\n",
      "test H@5: 0.5214, N@5: 0.4031, M@5: 0.3640\n",
      "test H@10: 0.6233, N@10: 0.4361, M@10: 0.3776\n",
      "test H@20: 0.7149, N@20: 0.4593, M@20: 0.3840\n",
      "\n",
      "Epoch [183/500], Train Loss: 17.0807, Elapsed: train 0.41 test 1.79\n",
      "valid H@5: 0.5064, N@5: 0.3919, M@5: 0.3538\n",
      "valid H@10: 0.6080, N@10: 0.4249, M@10: 0.3675\n",
      "valid H@20: 0.7013, N@20: 0.4485, M@20: 0.3741\n",
      "\n",
      "test H@5: 0.5219, N@5: 0.4036, M@5: 0.3645\n",
      "test H@10: 0.6221, N@10: 0.4361, M@10: 0.3779\n",
      "test H@20: 0.7149, N@20: 0.4596, M@20: 0.3844\n",
      "\n",
      "Epoch [184/500], Train Loss: 16.7879, Elapsed: train 0.51 test 1.92 *\n",
      "valid H@5: 0.5072, N@5: 0.3913, M@5: 0.3529\n",
      "valid H@10: 0.6084, N@10: 0.4241, M@10: 0.3665\n",
      "valid H@20: 0.7024, N@20: 0.4479, M@20: 0.3730\n",
      "\n",
      "test H@5: 0.5206, N@5: 0.4034, M@5: 0.3646\n",
      "test H@10: 0.6200, N@10: 0.4357, M@10: 0.3781\n",
      "test H@20: 0.7168, N@20: 0.4603, M@20: 0.3848\n",
      "\n",
      "Epoch [185/500], Train Loss: 16.8400, Elapsed: train 0.42 test 1.81 *\n",
      "valid H@5: 0.5070, N@5: 0.3918, M@5: 0.3535\n",
      "valid H@10: 0.6097, N@10: 0.4250, M@10: 0.3673\n",
      "valid H@20: 0.7009, N@20: 0.4481, M@20: 0.3736\n",
      "\n",
      "test H@5: 0.5198, N@5: 0.4022, M@5: 0.3632\n",
      "test H@10: 0.6220, N@10: 0.4354, M@10: 0.3770\n",
      "test H@20: 0.7166, N@20: 0.4595, M@20: 0.3837\n",
      "\n",
      "Epoch [186/500], Train Loss: 17.0232, Elapsed: train 0.46 test 1.96\n",
      "valid H@5: 0.5068, N@5: 0.3934, M@5: 0.3558\n",
      "valid H@10: 0.6080, N@10: 0.4263, M@10: 0.3694\n",
      "valid H@20: 0.7017, N@20: 0.4501, M@20: 0.3760\n",
      "\n",
      "test H@5: 0.5183, N@5: 0.4019, M@5: 0.3632\n",
      "test H@10: 0.6214, N@10: 0.4354, M@10: 0.3772\n",
      "test H@20: 0.7158, N@20: 0.4593, M@20: 0.3838\n",
      "\n",
      "Epoch [187/500], Train Loss: 16.8979, Elapsed: train 0.51 test 1.80\n",
      "valid H@5: 0.5072, N@5: 0.3930, M@5: 0.3549\n",
      "valid H@10: 0.6068, N@10: 0.4253, M@10: 0.3684\n",
      "valid H@20: 0.7005, N@20: 0.4491, M@20: 0.3749\n",
      "\n",
      "test H@5: 0.5187, N@5: 0.4022, M@5: 0.3636\n",
      "test H@10: 0.6200, N@10: 0.4351, M@10: 0.3773\n",
      "test H@20: 0.7164, N@20: 0.4596, M@20: 0.3840\n",
      "\n",
      "Epoch [188/500], Train Loss: 16.8607, Elapsed: train 0.46 test 1.78 *\n",
      "valid H@5: 0.5057, N@5: 0.3936, M@5: 0.3563\n",
      "valid H@10: 0.6099, N@10: 0.4273, M@10: 0.3703\n",
      "valid H@20: 0.7007, N@20: 0.4503, M@20: 0.3766\n",
      "\n",
      "test H@5: 0.5191, N@5: 0.4020, M@5: 0.3632\n",
      "test H@10: 0.6193, N@10: 0.4346, M@10: 0.3767\n",
      "test H@20: 0.7170, N@20: 0.4594, M@20: 0.3836\n",
      "\n",
      "Epoch [189/500], Train Loss: 16.8245, Elapsed: train 0.47 test 1.82 *\n",
      "valid H@5: 0.5093, N@5: 0.3941, M@5: 0.3559\n",
      "valid H@10: 0.6120, N@10: 0.4271, M@10: 0.3694\n",
      "valid H@20: 0.7001, N@20: 0.4493, M@20: 0.3754\n",
      "\n",
      "test H@5: 0.5202, N@5: 0.4024, M@5: 0.3633\n",
      "test H@10: 0.6221, N@10: 0.4354, M@10: 0.3770\n",
      "test H@20: 0.7168, N@20: 0.4595, M@20: 0.3837\n",
      "\n",
      "Epoch [190/500], Train Loss: 16.9875, Elapsed: train 0.53 test 1.83\n",
      "valid H@5: 0.5093, N@5: 0.3951, M@5: 0.3572\n",
      "valid H@10: 0.6114, N@10: 0.4280, M@10: 0.3707\n",
      "valid H@20: 0.7007, N@20: 0.4506, M@20: 0.3769\n",
      "\n",
      "test H@5: 0.5196, N@5: 0.4029, M@5: 0.3642\n",
      "test H@10: 0.6197, N@10: 0.4354, M@10: 0.3777\n",
      "test H@20: 0.7191, N@20: 0.4607, M@20: 0.3847\n",
      "\n",
      "Epoch [191/500], Train Loss: 16.7712, Elapsed: train 0.48 test 2.03\n",
      "valid H@5: 0.5095, N@5: 0.3940, M@5: 0.3557\n",
      "valid H@10: 0.6095, N@10: 0.4263, M@10: 0.3690\n",
      "valid H@20: 0.7017, N@20: 0.4496, M@20: 0.3754\n",
      "\n",
      "test H@5: 0.5200, N@5: 0.4035, M@5: 0.3649\n",
      "test H@10: 0.6218, N@10: 0.4365, M@10: 0.3785\n",
      "test H@20: 0.7189, N@20: 0.4612, M@20: 0.3853\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [192/500], Train Loss: 16.6973, Elapsed: train 0.48 test 1.74 *\n",
      "valid H@5: 0.5101, N@5: 0.3945, M@5: 0.3562\n",
      "valid H@10: 0.6118, N@10: 0.4273, M@10: 0.3697\n",
      "valid H@20: 0.7007, N@20: 0.4498, M@20: 0.3759\n",
      "\n",
      "test H@5: 0.5194, N@5: 0.4026, M@5: 0.3638\n",
      "test H@10: 0.6256, N@10: 0.4370, M@10: 0.3781\n",
      "test H@20: 0.7158, N@20: 0.4599, M@20: 0.3844\n",
      "\n",
      "Epoch [193/500], Train Loss: 16.6581, Elapsed: train 0.46 test 2.01 *\n",
      "valid H@5: 0.5108, N@5: 0.3945, M@5: 0.3560\n",
      "valid H@10: 0.6122, N@10: 0.4271, M@10: 0.3693\n",
      "valid H@20: 0.7009, N@20: 0.4495, M@20: 0.3754\n",
      "\n",
      "test H@5: 0.5173, N@5: 0.4016, M@5: 0.3632\n",
      "test H@10: 0.6266, N@10: 0.4371, M@10: 0.3779\n",
      "test H@20: 0.7162, N@20: 0.4599, M@20: 0.3842\n",
      "\n",
      "Epoch [194/500], Train Loss: 16.7940, Elapsed: train 0.56 test 1.85 *\n",
      "valid H@5: 0.5081, N@5: 0.3950, M@5: 0.3574\n",
      "valid H@10: 0.6128, N@10: 0.4288, M@10: 0.3713\n",
      "valid H@20: 0.7024, N@20: 0.4514, M@20: 0.3775\n",
      "\n",
      "test H@5: 0.5214, N@5: 0.4047, M@5: 0.3660\n",
      "test H@10: 0.6266, N@10: 0.4388, M@10: 0.3801\n",
      "test H@20: 0.7172, N@20: 0.4618, M@20: 0.3865\n",
      "\n",
      "Epoch [195/500], Train Loss: 16.8509, Elapsed: train 0.44 test 1.81\n",
      "valid H@5: 0.5093, N@5: 0.3948, M@5: 0.3568\n",
      "valid H@10: 0.6126, N@10: 0.4283, M@10: 0.3707\n",
      "valid H@20: 0.7021, N@20: 0.4509, M@20: 0.3769\n",
      "\n",
      "test H@5: 0.5223, N@5: 0.4050, M@5: 0.3662\n",
      "test H@10: 0.6254, N@10: 0.4385, M@10: 0.3800\n",
      "test H@20: 0.7164, N@20: 0.4617, M@20: 0.3865\n",
      "\n",
      "Epoch [196/500], Train Loss: 16.7998, Elapsed: train 0.47 test 1.90 *\n",
      "valid H@5: 0.5116, N@5: 0.3955, M@5: 0.3570\n",
      "valid H@10: 0.6103, N@10: 0.4274, M@10: 0.3702\n",
      "valid H@20: 0.7032, N@20: 0.4509, M@20: 0.3766\n",
      "\n",
      "test H@5: 0.5216, N@5: 0.4046, M@5: 0.3659\n",
      "test H@10: 0.6266, N@10: 0.4386, M@10: 0.3799\n",
      "test H@20: 0.7191, N@20: 0.4620, M@20: 0.3864\n",
      "\n",
      "Epoch [197/500], Train Loss: 16.9177, Elapsed: train 0.55 test 1.75 *\n",
      "valid H@5: 0.5122, N@5: 0.3956, M@5: 0.3570\n",
      "valid H@10: 0.6114, N@10: 0.4277, M@10: 0.3701\n",
      "valid H@20: 0.7026, N@20: 0.4507, M@20: 0.3765\n",
      "\n",
      "test H@5: 0.5198, N@5: 0.4038, M@5: 0.3653\n",
      "test H@10: 0.6237, N@10: 0.4375, M@10: 0.3793\n",
      "test H@20: 0.7204, N@20: 0.4621, M@20: 0.3861\n",
      "\n",
      "Epoch [198/500], Train Loss: 16.6905, Elapsed: train 0.49 test 1.83\n",
      "valid H@5: 0.5106, N@5: 0.3968, M@5: 0.3590\n",
      "valid H@10: 0.6110, N@10: 0.4292, M@10: 0.3723\n",
      "valid H@20: 0.7026, N@20: 0.4524, M@20: 0.3787\n",
      "\n",
      "test H@5: 0.5217, N@5: 0.4056, M@5: 0.3671\n",
      "test H@10: 0.6227, N@10: 0.4385, M@10: 0.3808\n",
      "test H@20: 0.7181, N@20: 0.4628, M@20: 0.3875\n",
      "\n",
      "Epoch [199/500], Train Loss: 16.6844, Elapsed: train 0.49 test 1.85\n",
      "valid H@5: 0.5110, N@5: 0.3953, M@5: 0.3568\n",
      "valid H@10: 0.6097, N@10: 0.4273, M@10: 0.3701\n",
      "valid H@20: 0.7030, N@20: 0.4510, M@20: 0.3766\n",
      "\n",
      "test H@5: 0.5237, N@5: 0.4057, M@5: 0.3666\n",
      "test H@10: 0.6233, N@10: 0.4381, M@10: 0.3801\n",
      "test H@20: 0.7189, N@20: 0.4624, M@20: 0.3868\n",
      "\n",
      "Epoch [200/500], Train Loss: 16.6888, Elapsed: train 0.51 test 1.81 *\n",
      "valid H@5: 0.5085, N@5: 0.3944, M@5: 0.3565\n",
      "valid H@10: 0.6105, N@10: 0.4275, M@10: 0.3702\n",
      "valid H@20: 0.7055, N@20: 0.4515, M@20: 0.3768\n",
      "\n",
      "test H@5: 0.5242, N@5: 0.4065, M@5: 0.3675\n",
      "test H@10: 0.6250, N@10: 0.4392, M@10: 0.3811\n",
      "test H@20: 0.7201, N@20: 0.4633, M@20: 0.3877\n",
      "\n",
      "Epoch [201/500], Train Loss: 16.7013, Elapsed: train 0.57 test 1.93 *\n",
      "valid H@5: 0.5104, N@5: 0.3956, M@5: 0.3575\n",
      "valid H@10: 0.6093, N@10: 0.4277, M@10: 0.3708\n",
      "valid H@20: 0.7057, N@20: 0.4521, M@20: 0.3775\n",
      "\n",
      "test H@5: 0.5260, N@5: 0.4079, M@5: 0.3689\n",
      "test H@10: 0.6262, N@10: 0.4403, M@10: 0.3822\n",
      "test H@20: 0.7187, N@20: 0.4638, M@20: 0.3887\n",
      "\n",
      "Epoch [202/500], Train Loss: 16.8375, Elapsed: train 0.54 test 1.91\n",
      "valid H@5: 0.5097, N@5: 0.3950, M@5: 0.3568\n",
      "valid H@10: 0.6089, N@10: 0.4272, M@10: 0.3702\n",
      "valid H@20: 0.7036, N@20: 0.4511, M@20: 0.3768\n",
      "\n",
      "test H@5: 0.5260, N@5: 0.4077, M@5: 0.3686\n",
      "test H@10: 0.6254, N@10: 0.4399, M@10: 0.3819\n",
      "test H@20: 0.7176, N@20: 0.4634, M@20: 0.3884\n",
      "\n",
      "Epoch [203/500], Train Loss: 16.7186, Elapsed: train 0.47 test 1.77\n",
      "valid H@5: 0.5103, N@5: 0.3953, M@5: 0.3572\n",
      "valid H@10: 0.6110, N@10: 0.4280, M@10: 0.3707\n",
      "valid H@20: 0.7045, N@20: 0.4516, M@20: 0.3771\n",
      "\n",
      "test H@5: 0.5242, N@5: 0.4063, M@5: 0.3672\n",
      "test H@10: 0.6250, N@10: 0.4391, M@10: 0.3808\n",
      "test H@20: 0.7199, N@20: 0.4631, M@20: 0.3875\n",
      "\n",
      "Epoch [204/500], Train Loss: 16.8682, Elapsed: train 0.51 test 1.75\n",
      "valid H@5: 0.5112, N@5: 0.3942, M@5: 0.3554\n",
      "valid H@10: 0.6120, N@10: 0.4268, M@10: 0.3688\n",
      "valid H@20: 0.7049, N@20: 0.4503, M@20: 0.3752\n",
      "\n",
      "test H@5: 0.5242, N@5: 0.4057, M@5: 0.3664\n",
      "test H@10: 0.6266, N@10: 0.4389, M@10: 0.3801\n",
      "test H@20: 0.7201, N@20: 0.4626, M@20: 0.3866\n",
      "\n",
      "Epoch [205/500], Train Loss: 16.6006, Elapsed: train 0.48 test 1.90\n",
      "valid H@5: 0.5106, N@5: 0.3922, M@5: 0.3529\n",
      "valid H@10: 0.6105, N@10: 0.4246, M@10: 0.3663\n",
      "valid H@20: 0.7028, N@20: 0.4480, M@20: 0.3727\n",
      "\n",
      "test H@5: 0.5223, N@5: 0.4054, M@5: 0.3665\n",
      "test H@10: 0.6243, N@10: 0.4386, M@10: 0.3804\n",
      "test H@20: 0.7191, N@20: 0.4626, M@20: 0.3870\n",
      "\n",
      "Epoch [206/500], Train Loss: 16.7375, Elapsed: train 0.50 test 1.76\n",
      "valid H@5: 0.5078, N@5: 0.3926, M@5: 0.3544\n",
      "valid H@10: 0.6107, N@10: 0.4260, M@10: 0.3682\n",
      "valid H@20: 0.7003, N@20: 0.4487, M@20: 0.3745\n",
      "\n",
      "test H@5: 0.5248, N@5: 0.4061, M@5: 0.3668\n",
      "test H@10: 0.6254, N@10: 0.4387, M@10: 0.3803\n",
      "test H@20: 0.7185, N@20: 0.4624, M@20: 0.3868\n",
      "\n",
      "Epoch [207/500], Train Loss: 16.5186, Elapsed: train 0.52 test 1.75\n",
      "valid H@5: 0.5093, N@5: 0.3930, M@5: 0.3544\n",
      "valid H@10: 0.6112, N@10: 0.4259, M@10: 0.3679\n",
      "valid H@20: 0.7007, N@20: 0.4487, M@20: 0.3742\n",
      "\n",
      "test H@5: 0.5262, N@5: 0.4060, M@5: 0.3661\n",
      "test H@10: 0.6258, N@10: 0.4383, M@10: 0.3795\n",
      "test H@20: 0.7206, N@20: 0.4623, M@20: 0.3862\n",
      "\n",
      "Epoch [208/500], Train Loss: 16.5547, Elapsed: train 0.47 test 1.79\n",
      "valid H@5: 0.5081, N@5: 0.3927, M@5: 0.3543\n",
      "valid H@10: 0.6114, N@10: 0.4260, M@10: 0.3681\n",
      "valid H@20: 0.7009, N@20: 0.4488, M@20: 0.3744\n",
      "\n",
      "test H@5: 0.5242, N@5: 0.4056, M@5: 0.3663\n",
      "test H@10: 0.6271, N@10: 0.4391, M@10: 0.3802\n",
      "test H@20: 0.7210, N@20: 0.4628, M@20: 0.3867\n",
      "\n",
      "Epoch [209/500], Train Loss: 16.8571, Elapsed: train 0.47 test 1.81 *\n",
      "valid H@5: 0.5066, N@5: 0.3931, M@5: 0.3553\n",
      "valid H@10: 0.6135, N@10: 0.4277, M@10: 0.3696\n",
      "valid H@20: 0.7021, N@20: 0.4502, M@20: 0.3759\n",
      "\n",
      "test H@5: 0.5248, N@5: 0.4063, M@5: 0.3670\n",
      "test H@10: 0.6254, N@10: 0.4390, M@10: 0.3806\n",
      "test H@20: 0.7195, N@20: 0.4629, M@20: 0.3872\n",
      "\n",
      "Epoch [210/500], Train Loss: 16.5266, Elapsed: train 0.53 test 1.80 *\n",
      "valid H@5: 0.5078, N@5: 0.3943, M@5: 0.3566\n",
      "valid H@10: 0.6139, N@10: 0.4287, M@10: 0.3708\n",
      "valid H@20: 0.7005, N@20: 0.4507, M@20: 0.3769\n",
      "\n",
      "test H@5: 0.5258, N@5: 0.4075, M@5: 0.3683\n",
      "test H@10: 0.6279, N@10: 0.4406, M@10: 0.3820\n",
      "test H@20: 0.7206, N@20: 0.4642, M@20: 0.3885\n",
      "\n",
      "Epoch [211/500], Train Loss: 16.6202, Elapsed: train 0.45 test 1.83\n",
      "valid H@5: 0.5085, N@5: 0.3935, M@5: 0.3552\n",
      "valid H@10: 0.6118, N@10: 0.4270, M@10: 0.3691\n",
      "valid H@20: 0.7022, N@20: 0.4500, M@20: 0.3754\n",
      "\n",
      "test H@5: 0.5256, N@5: 0.4087, M@5: 0.3699\n",
      "test H@10: 0.6254, N@10: 0.4412, M@10: 0.3834\n",
      "test H@20: 0.7206, N@20: 0.4653, M@20: 0.3901\n",
      "\n",
      "Epoch [212/500], Train Loss: 16.5290, Elapsed: train 0.48 test 1.84\n",
      "valid H@5: 0.5089, N@5: 0.3952, M@5: 0.3573\n",
      "valid H@10: 0.6124, N@10: 0.4285, M@10: 0.3710\n",
      "valid H@20: 0.7022, N@20: 0.4513, M@20: 0.3773\n",
      "\n",
      "test H@5: 0.5265, N@5: 0.4090, M@5: 0.3700\n",
      "test H@10: 0.6262, N@10: 0.4415, M@10: 0.3835\n",
      "test H@20: 0.7199, N@20: 0.4653, M@20: 0.3901\n",
      "\n",
      "Epoch [213/500], Train Loss: 16.6194, Elapsed: train 0.47 test 1.83\n",
      "valid H@5: 0.5091, N@5: 0.3962, M@5: 0.3586\n",
      "valid H@10: 0.6089, N@10: 0.4286, M@10: 0.3721\n",
      "valid H@20: 0.7003, N@20: 0.4518, M@20: 0.3785\n",
      "\n",
      "test H@5: 0.5269, N@5: 0.4091, M@5: 0.3701\n",
      "test H@10: 0.6256, N@10: 0.4412, M@10: 0.3834\n",
      "test H@20: 0.7204, N@20: 0.4653, M@20: 0.3900\n",
      "\n",
      "Epoch [214/500], Train Loss: 16.4085, Elapsed: train 0.50 test 1.75\n",
      "valid H@5: 0.5072, N@5: 0.3946, M@5: 0.3573\n",
      "valid H@10: 0.6122, N@10: 0.4287, M@10: 0.3714\n",
      "valid H@20: 0.6988, N@20: 0.4507, M@20: 0.3775\n",
      "\n",
      "test H@5: 0.5250, N@5: 0.4088, M@5: 0.3703\n",
      "test H@10: 0.6264, N@10: 0.4418, M@10: 0.3840\n",
      "test H@20: 0.7197, N@20: 0.4655, M@20: 0.3906\n",
      "\n",
      "Epoch [215/500], Train Loss: 16.5020, Elapsed: train 0.52 test 1.95\n",
      "valid H@5: 0.5089, N@5: 0.3953, M@5: 0.3576\n",
      "valid H@10: 0.6120, N@10: 0.4288, M@10: 0.3715\n",
      "valid H@20: 0.6998, N@20: 0.4511, M@20: 0.3777\n",
      "\n",
      "test H@5: 0.5229, N@5: 0.4078, M@5: 0.3696\n",
      "test H@10: 0.6256, N@10: 0.4412, M@10: 0.3835\n",
      "test H@20: 0.7197, N@20: 0.4652, M@20: 0.3902\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [216/500], Train Loss: 16.5284, Elapsed: train 0.47 test 1.76\n",
      "valid H@5: 0.5074, N@5: 0.3940, M@5: 0.3563\n",
      "valid H@10: 0.6122, N@10: 0.4280, M@10: 0.3704\n",
      "valid H@20: 0.6992, N@20: 0.4501, M@20: 0.3766\n",
      "\n",
      "test H@5: 0.5254, N@5: 0.4078, M@5: 0.3688\n",
      "test H@10: 0.6258, N@10: 0.4405, M@10: 0.3824\n",
      "test H@20: 0.7193, N@20: 0.4643, M@20: 0.3890\n",
      "\n",
      "Epoch [217/500], Train Loss: 16.6604, Elapsed: train 0.55 test 1.78\n",
      "valid H@5: 0.5078, N@5: 0.3935, M@5: 0.3555\n",
      "valid H@10: 0.6107, N@10: 0.4268, M@10: 0.3693\n",
      "valid H@20: 0.6988, N@20: 0.4492, M@20: 0.3755\n",
      "\n",
      "test H@5: 0.5252, N@5: 0.4078, M@5: 0.3689\n",
      "test H@10: 0.6271, N@10: 0.4408, M@10: 0.3825\n",
      "test H@20: 0.7214, N@20: 0.4647, M@20: 0.3891\n",
      "\n",
      "Epoch [218/500], Train Loss: 16.5670, Elapsed: train 0.47 test 1.79\n",
      "valid H@5: 0.5078, N@5: 0.3948, M@5: 0.3573\n",
      "valid H@10: 0.6108, N@10: 0.4282, M@10: 0.3711\n",
      "valid H@20: 0.6994, N@20: 0.4508, M@20: 0.3774\n",
      "\n",
      "test H@5: 0.5240, N@5: 0.4084, M@5: 0.3700\n",
      "test H@10: 0.6264, N@10: 0.4415, M@10: 0.3837\n",
      "test H@20: 0.7220, N@20: 0.4657, M@20: 0.3904\n",
      "\n",
      "Epoch [219/500], Train Loss: 16.5499, Elapsed: train 0.45 test 1.79\n",
      "valid H@5: 0.5095, N@5: 0.3963, M@5: 0.3587\n",
      "valid H@10: 0.6114, N@10: 0.4293, M@10: 0.3723\n",
      "valid H@20: 0.7005, N@20: 0.4519, M@20: 0.3786\n",
      "\n",
      "test H@5: 0.5275, N@5: 0.4093, M@5: 0.3701\n",
      "test H@10: 0.6275, N@10: 0.4417, M@10: 0.3835\n",
      "test H@20: 0.7216, N@20: 0.4654, M@20: 0.3900\n",
      "\n",
      "Epoch [220/500], Train Loss: 16.6732, Elapsed: train 0.52 test 1.75\n",
      "valid H@5: 0.5093, N@5: 0.3974, M@5: 0.3602\n",
      "valid H@10: 0.6118, N@10: 0.4305, M@10: 0.3738\n",
      "valid H@20: 0.7007, N@20: 0.4532, M@20: 0.3801\n",
      "\n",
      "test H@5: 0.5275, N@5: 0.4092, M@5: 0.3699\n",
      "test H@10: 0.6292, N@10: 0.4421, M@10: 0.3834\n",
      "test H@20: 0.7216, N@20: 0.4654, M@20: 0.3898\n",
      "\n",
      "Epoch [221/500], Train Loss: 16.6384, Elapsed: train 0.57 test 1.79\n",
      "valid H@5: 0.5093, N@5: 0.3972, M@5: 0.3598\n",
      "valid H@10: 0.6101, N@10: 0.4298, M@10: 0.3733\n",
      "valid H@20: 0.7024, N@20: 0.4534, M@20: 0.3799\n",
      "\n",
      "test H@5: 0.5292, N@5: 0.4101, M@5: 0.3706\n",
      "test H@10: 0.6289, N@10: 0.4422, M@10: 0.3838\n",
      "test H@20: 0.7216, N@20: 0.4657, M@20: 0.3903\n",
      "\n",
      "Epoch [222/500], Train Loss: 16.4880, Elapsed: train 0.48 test 1.76\n",
      "valid H@5: 0.5097, N@5: 0.3972, M@5: 0.3597\n",
      "valid H@10: 0.6122, N@10: 0.4304, M@10: 0.3735\n",
      "valid H@20: 0.7045, N@20: 0.4538, M@20: 0.3799\n",
      "\n",
      "test H@5: 0.5290, N@5: 0.4112, M@5: 0.3722\n",
      "test H@10: 0.6285, N@10: 0.4434, M@10: 0.3854\n",
      "test H@20: 0.7204, N@20: 0.4666, M@20: 0.3918\n",
      "\n",
      "Epoch [223/500], Train Loss: 16.4597, Elapsed: train 0.49 test 1.79\n",
      "valid H@5: 0.5099, N@5: 0.3957, M@5: 0.3578\n",
      "valid H@10: 0.6130, N@10: 0.4292, M@10: 0.3717\n",
      "valid H@20: 0.7024, N@20: 0.4519, M@20: 0.3780\n",
      "\n",
      "test H@5: 0.5271, N@5: 0.4098, M@5: 0.3709\n",
      "test H@10: 0.6244, N@10: 0.4415, M@10: 0.3841\n",
      "test H@20: 0.7197, N@20: 0.4657, M@20: 0.3908\n",
      "\n",
      "Epoch [224/500], Train Loss: 16.5267, Elapsed: train 0.46 test 1.75 *\n",
      "valid H@5: 0.5126, N@5: 0.3963, M@5: 0.3576\n",
      "valid H@10: 0.6122, N@10: 0.4285, M@10: 0.3710\n",
      "valid H@20: 0.7019, N@20: 0.4514, M@20: 0.3773\n",
      "\n",
      "test H@5: 0.5300, N@5: 0.4111, M@5: 0.3717\n",
      "test H@10: 0.6241, N@10: 0.4416, M@10: 0.3843\n",
      "test H@20: 0.7195, N@20: 0.4659, M@20: 0.3911\n",
      "\n",
      "Epoch [225/500], Train Loss: 16.4911, Elapsed: train 0.50 test 1.77\n",
      "valid H@5: 0.5104, N@5: 0.3961, M@5: 0.3580\n",
      "valid H@10: 0.6128, N@10: 0.4292, M@10: 0.3718\n",
      "valid H@20: 0.7007, N@20: 0.4516, M@20: 0.3780\n",
      "\n",
      "test H@5: 0.5271, N@5: 0.4102, M@5: 0.3715\n",
      "test H@10: 0.6269, N@10: 0.4426, M@10: 0.3849\n",
      "test H@20: 0.7204, N@20: 0.4663, M@20: 0.3914\n",
      "\n",
      "Epoch [226/500], Train Loss: 16.4053, Elapsed: train 0.47 test 1.95 *\n",
      "valid H@5: 0.5112, N@5: 0.3968, M@5: 0.3588\n",
      "valid H@10: 0.6154, N@10: 0.4305, M@10: 0.3727\n",
      "valid H@20: 0.7013, N@20: 0.4523, M@20: 0.3787\n",
      "\n",
      "test H@5: 0.5244, N@5: 0.4088, M@5: 0.3705\n",
      "test H@10: 0.6275, N@10: 0.4423, M@10: 0.3844\n",
      "test H@20: 0.7206, N@20: 0.4660, M@20: 0.3910\n",
      "\n",
      "Epoch [227/500], Train Loss: 16.4901, Elapsed: train 0.52 test 1.80\n",
      "valid H@5: 0.5114, N@5: 0.3963, M@5: 0.3581\n",
      "valid H@10: 0.6147, N@10: 0.4298, M@10: 0.3719\n",
      "valid H@20: 0.7028, N@20: 0.4522, M@20: 0.3781\n",
      "\n",
      "test H@5: 0.5258, N@5: 0.4088, M@5: 0.3700\n",
      "test H@10: 0.6300, N@10: 0.4425, M@10: 0.3839\n",
      "test H@20: 0.7199, N@20: 0.4653, M@20: 0.3902\n",
      "\n",
      "Epoch [228/500], Train Loss: 16.6546, Elapsed: train 0.47 test 1.74\n",
      "valid H@5: 0.5108, N@5: 0.3968, M@5: 0.3589\n",
      "valid H@10: 0.6139, N@10: 0.4302, M@10: 0.3728\n",
      "valid H@20: 0.7036, N@20: 0.4530, M@20: 0.3790\n",
      "\n",
      "test H@5: 0.5267, N@5: 0.4096, M@5: 0.3708\n",
      "test H@10: 0.6262, N@10: 0.4418, M@10: 0.3841\n",
      "test H@20: 0.7220, N@20: 0.4662, M@20: 0.3908\n",
      "\n",
      "Epoch [229/500], Train Loss: 16.4823, Elapsed: train 0.46 test 1.82\n",
      "valid H@5: 0.5110, N@5: 0.3973, M@5: 0.3595\n",
      "valid H@10: 0.6143, N@10: 0.4308, M@10: 0.3734\n",
      "valid H@20: 0.7051, N@20: 0.4537, M@20: 0.3797\n",
      "\n",
      "test H@5: 0.5262, N@5: 0.4098, M@5: 0.3712\n",
      "test H@10: 0.6298, N@10: 0.4433, M@10: 0.3850\n",
      "test H@20: 0.7214, N@20: 0.4665, M@20: 0.3914\n",
      "\n",
      "Epoch [230/500], Train Loss: 16.5730, Elapsed: train 0.55 test 1.78\n",
      "valid H@5: 0.5076, N@5: 0.3954, M@5: 0.3580\n",
      "valid H@10: 0.6128, N@10: 0.4297, M@10: 0.3724\n",
      "valid H@20: 0.7028, N@20: 0.4525, M@20: 0.3787\n",
      "\n",
      "test H@5: 0.5231, N@5: 0.4078, M@5: 0.3695\n",
      "test H@10: 0.6312, N@10: 0.4428, M@10: 0.3841\n",
      "test H@20: 0.7214, N@20: 0.4657, M@20: 0.3903\n",
      "\n",
      "Epoch [231/500], Train Loss: 16.6425, Elapsed: train 0.46 test 1.90 *\n",
      "valid H@5: 0.5127, N@5: 0.3962, M@5: 0.3575\n",
      "valid H@10: 0.6120, N@10: 0.4284, M@10: 0.3708\n",
      "valid H@20: 0.7030, N@20: 0.4515, M@20: 0.3772\n",
      "\n",
      "test H@5: 0.5233, N@5: 0.4070, M@5: 0.3685\n",
      "test H@10: 0.6300, N@10: 0.4416, M@10: 0.3828\n",
      "test H@20: 0.7206, N@20: 0.4645, M@20: 0.3891\n",
      "\n",
      "Epoch [232/500], Train Loss: 16.4483, Elapsed: train 0.47 test 1.90\n",
      "valid H@5: 0.5112, N@5: 0.3956, M@5: 0.3572\n",
      "valid H@10: 0.6120, N@10: 0.4283, M@10: 0.3708\n",
      "valid H@20: 0.7047, N@20: 0.4518, M@20: 0.3772\n",
      "\n",
      "test H@5: 0.5275, N@5: 0.4087, M@5: 0.3693\n",
      "test H@10: 0.6271, N@10: 0.4409, M@10: 0.3827\n",
      "test H@20: 0.7201, N@20: 0.4645, M@20: 0.3892\n",
      "\n",
      "Epoch [233/500], Train Loss: 16.4301, Elapsed: train 0.47 test 1.78\n",
      "valid H@5: 0.5099, N@5: 0.3960, M@5: 0.3582\n",
      "valid H@10: 0.6128, N@10: 0.4293, M@10: 0.3719\n",
      "valid H@20: 0.7038, N@20: 0.4523, M@20: 0.3782\n",
      "\n",
      "test H@5: 0.5239, N@5: 0.4066, M@5: 0.3678\n",
      "test H@10: 0.6264, N@10: 0.4400, M@10: 0.3816\n",
      "test H@20: 0.7206, N@20: 0.4639, M@20: 0.3883\n",
      "\n",
      "Epoch [234/500], Train Loss: 16.2553, Elapsed: train 0.54 test 1.81 *\n",
      "valid H@5: 0.5103, N@5: 0.3955, M@5: 0.3573\n",
      "valid H@10: 0.6112, N@10: 0.4283, M@10: 0.3710\n",
      "valid H@20: 0.7065, N@20: 0.4525, M@20: 0.3777\n",
      "\n",
      "test H@5: 0.5248, N@5: 0.4071, M@5: 0.3681\n",
      "test H@10: 0.6269, N@10: 0.4404, M@10: 0.3820\n",
      "test H@20: 0.7224, N@20: 0.4645, M@20: 0.3886\n",
      "\n",
      "Epoch [235/500], Train Loss: 16.3621, Elapsed: train 0.49 test 1.81 *\n",
      "valid H@5: 0.5104, N@5: 0.3952, M@5: 0.3568\n",
      "valid H@10: 0.6141, N@10: 0.4288, M@10: 0.3707\n",
      "valid H@20: 0.7068, N@20: 0.4523, M@20: 0.3772\n",
      "\n",
      "test H@5: 0.5229, N@5: 0.4066, M@5: 0.3680\n",
      "test H@10: 0.6271, N@10: 0.4404, M@10: 0.3821\n",
      "test H@20: 0.7208, N@20: 0.4642, M@20: 0.3886\n",
      "\n",
      "Epoch [236/500], Train Loss: 16.4119, Elapsed: train 0.49 test 1.86\n",
      "valid H@5: 0.5101, N@5: 0.3947, M@5: 0.3563\n",
      "valid H@10: 0.6130, N@10: 0.4280, M@10: 0.3701\n",
      "valid H@20: 0.7057, N@20: 0.4516, M@20: 0.3767\n",
      "\n",
      "test H@5: 0.5262, N@5: 0.4086, M@5: 0.3697\n",
      "test H@10: 0.6273, N@10: 0.4415, M@10: 0.3833\n",
      "test H@20: 0.7229, N@20: 0.4657, M@20: 0.3900\n",
      "\n",
      "Epoch [237/500], Train Loss: 16.3670, Elapsed: train 0.56 test 1.80 *\n",
      "valid H@5: 0.5131, N@5: 0.3964, M@5: 0.3576\n",
      "valid H@10: 0.6139, N@10: 0.4290, M@10: 0.3711\n",
      "valid H@20: 0.7057, N@20: 0.4523, M@20: 0.3775\n",
      "\n",
      "test H@5: 0.5254, N@5: 0.4080, M@5: 0.3690\n",
      "test H@10: 0.6266, N@10: 0.4408, M@10: 0.3827\n",
      "test H@20: 0.7231, N@20: 0.4653, M@20: 0.3894\n",
      "\n",
      "Epoch [238/500], Train Loss: 16.3472, Elapsed: train 0.45 test 1.77 *\n",
      "valid H@5: 0.5143, N@5: 0.3978, M@5: 0.3592\n",
      "valid H@10: 0.6128, N@10: 0.4297, M@10: 0.3724\n",
      "valid H@20: 0.7047, N@20: 0.4531, M@20: 0.3788\n",
      "\n",
      "test H@5: 0.5229, N@5: 0.4075, M@5: 0.3692\n",
      "test H@10: 0.6279, N@10: 0.4415, M@10: 0.3833\n",
      "test H@20: 0.7237, N@20: 0.4658, M@20: 0.3900\n",
      "\n",
      "Epoch [239/500], Train Loss: 16.2592, Elapsed: train 0.49 test 1.83\n",
      "valid H@5: 0.5110, N@5: 0.3955, M@5: 0.3572\n",
      "valid H@10: 0.6137, N@10: 0.4288, M@10: 0.3709\n",
      "valid H@20: 0.7043, N@20: 0.4518, M@20: 0.3773\n",
      "\n",
      "test H@5: 0.5212, N@5: 0.4075, M@5: 0.3697\n",
      "test H@10: 0.6273, N@10: 0.4420, M@10: 0.3841\n",
      "test H@20: 0.7224, N@20: 0.4661, M@20: 0.3908\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [240/500], Train Loss: 16.4269, Elapsed: train 0.55 test 1.91\n",
      "valid H@5: 0.5118, N@5: 0.3959, M@5: 0.3575\n",
      "valid H@10: 0.6153, N@10: 0.4294, M@10: 0.3713\n",
      "valid H@20: 0.7043, N@20: 0.4521, M@20: 0.3776\n",
      "\n",
      "test H@5: 0.5210, N@5: 0.4074, M@5: 0.3697\n",
      "test H@10: 0.6300, N@10: 0.4427, M@10: 0.3843\n",
      "test H@20: 0.7229, N@20: 0.4662, M@20: 0.3907\n",
      "\n",
      "Epoch [241/500], Train Loss: 16.2538, Elapsed: train 0.48 test 1.90\n",
      "valid H@5: 0.5095, N@5: 0.3953, M@5: 0.3573\n",
      "valid H@10: 0.6147, N@10: 0.4294, M@10: 0.3715\n",
      "valid H@20: 0.7057, N@20: 0.4525, M@20: 0.3778\n",
      "\n",
      "test H@5: 0.5233, N@5: 0.4090, M@5: 0.3711\n",
      "test H@10: 0.6285, N@10: 0.4430, M@10: 0.3851\n",
      "test H@20: 0.7256, N@20: 0.4676, M@20: 0.3919\n",
      "\n",
      "Epoch [242/500], Train Loss: 16.4864, Elapsed: train 0.52 test 1.80 *\n",
      "valid H@5: 0.5104, N@5: 0.3965, M@5: 0.3586\n",
      "valid H@10: 0.6160, N@10: 0.4307, M@10: 0.3728\n",
      "valid H@20: 0.7045, N@20: 0.4532, M@20: 0.3790\n",
      "\n",
      "test H@5: 0.5235, N@5: 0.4089, M@5: 0.3708\n",
      "test H@10: 0.6312, N@10: 0.4437, M@10: 0.3852\n",
      "test H@20: 0.7247, N@20: 0.4673, M@20: 0.3917\n",
      "\n",
      "Epoch [243/500], Train Loss: 16.5786, Elapsed: train 0.53 test 1.87 *\n",
      "valid H@5: 0.5110, N@5: 0.3955, M@5: 0.3571\n",
      "valid H@10: 0.6162, N@10: 0.4296, M@10: 0.3712\n",
      "valid H@20: 0.7051, N@20: 0.4521, M@20: 0.3774\n",
      "\n",
      "test H@5: 0.5252, N@5: 0.4101, M@5: 0.3720\n",
      "test H@10: 0.6292, N@10: 0.4439, M@10: 0.3859\n",
      "test H@20: 0.7243, N@20: 0.4679, M@20: 0.3925\n",
      "\n",
      "Epoch [244/500], Train Loss: 16.2994, Elapsed: train 0.46 test 1.96 *\n",
      "valid H@5: 0.5118, N@5: 0.3954, M@5: 0.3568\n",
      "valid H@10: 0.6172, N@10: 0.4295, M@10: 0.3708\n",
      "valid H@20: 0.7066, N@20: 0.4521, M@20: 0.3770\n",
      "\n",
      "test H@5: 0.5237, N@5: 0.4085, M@5: 0.3702\n",
      "test H@10: 0.6298, N@10: 0.4430, M@10: 0.3846\n",
      "test H@20: 0.7218, N@20: 0.4663, M@20: 0.3910\n",
      "\n",
      "Epoch [245/500], Train Loss: 16.3823, Elapsed: train 0.47 test 1.75 *\n",
      "valid H@5: 0.5085, N@5: 0.3957, M@5: 0.3583\n",
      "valid H@10: 0.6177, N@10: 0.4312, M@10: 0.3729\n",
      "valid H@20: 0.7061, N@20: 0.4535, M@20: 0.3791\n",
      "\n",
      "test H@5: 0.5231, N@5: 0.4085, M@5: 0.3705\n",
      "test H@10: 0.6300, N@10: 0.4434, M@10: 0.3850\n",
      "test H@20: 0.7206, N@20: 0.4664, M@20: 0.3914\n",
      "\n",
      "Epoch [246/500], Train Loss: 16.4857, Elapsed: train 0.49 test 1.81 *\n",
      "valid H@5: 0.5108, N@5: 0.3977, M@5: 0.3601\n",
      "valid H@10: 0.6185, N@10: 0.4325, M@10: 0.3745\n",
      "valid H@20: 0.7065, N@20: 0.4547, M@20: 0.3806\n",
      "\n",
      "test H@5: 0.5248, N@5: 0.4098, M@5: 0.3717\n",
      "test H@10: 0.6317, N@10: 0.4445, M@10: 0.3860\n",
      "test H@20: 0.7216, N@20: 0.4672, M@20: 0.3923\n",
      "\n",
      "Epoch [247/500], Train Loss: 16.3767, Elapsed: train 0.51 test 1.78 *\n",
      "valid H@5: 0.5103, N@5: 0.3972, M@5: 0.3596\n",
      "valid H@10: 0.6185, N@10: 0.4322, M@10: 0.3740\n",
      "valid H@20: 0.7084, N@20: 0.4549, M@20: 0.3802\n",
      "\n",
      "test H@5: 0.5242, N@5: 0.4083, M@5: 0.3698\n",
      "test H@10: 0.6312, N@10: 0.4431, M@10: 0.3843\n",
      "test H@20: 0.7204, N@20: 0.4657, M@20: 0.3905\n",
      "\n",
      "Epoch [248/500], Train Loss: 16.4486, Elapsed: train 0.46 test 1.82\n",
      "valid H@5: 0.5133, N@5: 0.3978, M@5: 0.3595\n",
      "valid H@10: 0.6172, N@10: 0.4313, M@10: 0.3732\n",
      "valid H@20: 0.7066, N@20: 0.4539, M@20: 0.3794\n",
      "\n",
      "test H@5: 0.5227, N@5: 0.4073, M@5: 0.3690\n",
      "test H@10: 0.6312, N@10: 0.4427, M@10: 0.3837\n",
      "test H@20: 0.7220, N@20: 0.4657, M@20: 0.3901\n",
      "\n",
      "Epoch [249/500], Train Loss: 16.2823, Elapsed: train 0.48 test 1.80\n",
      "valid H@5: 0.5116, N@5: 0.3976, M@5: 0.3596\n",
      "valid H@10: 0.6181, N@10: 0.4320, M@10: 0.3738\n",
      "valid H@20: 0.7080, N@20: 0.4548, M@20: 0.3801\n",
      "\n",
      "test H@5: 0.5246, N@5: 0.4094, M@5: 0.3711\n",
      "test H@10: 0.6325, N@10: 0.4444, M@10: 0.3857\n",
      "test H@20: 0.7224, N@20: 0.4672, M@20: 0.3920\n",
      "\n",
      "Epoch [250/500], Train Loss: 16.2524, Elapsed: train 0.53 test 1.80 *\n",
      "valid H@5: 0.5110, N@5: 0.3960, M@5: 0.3578\n",
      "valid H@10: 0.6187, N@10: 0.4308, M@10: 0.3721\n",
      "valid H@20: 0.7084, N@20: 0.4535, M@20: 0.3784\n",
      "\n",
      "test H@5: 0.5254, N@5: 0.4094, M@5: 0.3709\n",
      "test H@10: 0.6331, N@10: 0.4444, M@10: 0.3854\n",
      "test H@20: 0.7218, N@20: 0.4669, M@20: 0.3917\n",
      "\n",
      "Epoch [251/500], Train Loss: 16.2211, Elapsed: train 0.46 test 1.78\n",
      "valid H@5: 0.5103, N@5: 0.3952, M@5: 0.3569\n",
      "valid H@10: 0.6183, N@10: 0.4301, M@10: 0.3714\n",
      "valid H@20: 0.7082, N@20: 0.4530, M@20: 0.3777\n",
      "\n",
      "test H@5: 0.5256, N@5: 0.4092, M@5: 0.3705\n",
      "test H@10: 0.6329, N@10: 0.4439, M@10: 0.3849\n",
      "test H@20: 0.7229, N@20: 0.4667, M@20: 0.3912\n",
      "\n",
      "Epoch [252/500], Train Loss: 16.2299, Elapsed: train 0.45 test 1.81 *\n",
      "valid H@5: 0.5118, N@5: 0.3978, M@5: 0.3599\n",
      "valid H@10: 0.6204, N@10: 0.4329, M@10: 0.3743\n",
      "valid H@20: 0.7076, N@20: 0.4549, M@20: 0.3804\n",
      "\n",
      "test H@5: 0.5271, N@5: 0.4098, M@5: 0.3708\n",
      "test H@10: 0.6308, N@10: 0.4434, M@10: 0.3848\n",
      "test H@20: 0.7229, N@20: 0.4668, M@20: 0.3912\n",
      "\n",
      "Epoch [253/500], Train Loss: 16.3613, Elapsed: train 0.55 test 1.81\n",
      "valid H@5: 0.5129, N@5: 0.3982, M@5: 0.3601\n",
      "valid H@10: 0.6179, N@10: 0.4322, M@10: 0.3741\n",
      "valid H@20: 0.7051, N@20: 0.4544, M@20: 0.3803\n",
      "\n",
      "test H@5: 0.5281, N@5: 0.4100, M@5: 0.3709\n",
      "test H@10: 0.6304, N@10: 0.4431, M@10: 0.3845\n",
      "test H@20: 0.7208, N@20: 0.4661, M@20: 0.3909\n",
      "\n",
      "Epoch [254/500], Train Loss: 16.4025, Elapsed: train 0.49 test 1.79 *\n",
      "valid H@5: 0.5152, N@5: 0.3986, M@5: 0.3598\n",
      "valid H@10: 0.6158, N@10: 0.4312, M@10: 0.3734\n",
      "valid H@20: 0.7072, N@20: 0.4544, M@20: 0.3798\n",
      "\n",
      "test H@5: 0.5285, N@5: 0.4102, M@5: 0.3710\n",
      "test H@10: 0.6289, N@10: 0.4429, M@10: 0.3846\n",
      "test H@20: 0.7212, N@20: 0.4664, M@20: 0.3911\n",
      "\n",
      "Epoch [255/500], Train Loss: 16.0536, Elapsed: train 0.49 test 1.80\n",
      "valid H@5: 0.5147, N@5: 0.3990, M@5: 0.3605\n",
      "valid H@10: 0.6183, N@10: 0.4327, M@10: 0.3745\n",
      "valid H@20: 0.7078, N@20: 0.4553, M@20: 0.3808\n",
      "\n",
      "test H@5: 0.5283, N@5: 0.4097, M@5: 0.3703\n",
      "test H@10: 0.6312, N@10: 0.4431, M@10: 0.3842\n",
      "test H@20: 0.7216, N@20: 0.4660, M@20: 0.3905\n",
      "\n",
      "Epoch [256/500], Train Loss: 16.3432, Elapsed: train 0.46 test 1.81\n",
      "valid H@5: 0.5112, N@5: 0.3976, M@5: 0.3598\n",
      "valid H@10: 0.6181, N@10: 0.4324, M@10: 0.3743\n",
      "valid H@20: 0.7068, N@20: 0.4549, M@20: 0.3805\n",
      "\n",
      "test H@5: 0.5283, N@5: 0.4096, M@5: 0.3702\n",
      "test H@10: 0.6310, N@10: 0.4430, M@10: 0.3841\n",
      "test H@20: 0.7233, N@20: 0.4664, M@20: 0.3905\n",
      "\n",
      "Epoch [257/500], Train Loss: 16.3671, Elapsed: train 0.54 test 1.79\n",
      "valid H@5: 0.5124, N@5: 0.3977, M@5: 0.3595\n",
      "valid H@10: 0.6189, N@10: 0.4324, M@10: 0.3740\n",
      "valid H@20: 0.7074, N@20: 0.4548, M@20: 0.3802\n",
      "\n",
      "test H@5: 0.5250, N@5: 0.4086, M@5: 0.3700\n",
      "test H@10: 0.6302, N@10: 0.4430, M@10: 0.3843\n",
      "test H@20: 0.7237, N@20: 0.4667, M@20: 0.3909\n",
      "\n",
      "Epoch [258/500], Train Loss: 16.2183, Elapsed: train 0.46 test 1.77\n",
      "valid H@5: 0.5147, N@5: 0.3993, M@5: 0.3610\n",
      "valid H@10: 0.6179, N@10: 0.4329, M@10: 0.3750\n",
      "valid H@20: 0.7072, N@20: 0.4555, M@20: 0.3812\n",
      "\n",
      "test H@5: 0.5262, N@5: 0.4081, M@5: 0.3689\n",
      "test H@10: 0.6300, N@10: 0.4419, M@10: 0.3830\n",
      "test H@20: 0.7241, N@20: 0.4657, M@20: 0.3896\n",
      "\n",
      "Epoch [259/500], Train Loss: 16.2288, Elapsed: train 0.46 test 1.68 *\n",
      "valid H@5: 0.5137, N@5: 0.3979, M@5: 0.3594\n",
      "valid H@10: 0.6208, N@10: 0.4327, M@10: 0.3738\n",
      "valid H@20: 0.7084, N@20: 0.4548, M@20: 0.3799\n",
      "\n",
      "test H@5: 0.5275, N@5: 0.4094, M@5: 0.3702\n",
      "test H@10: 0.6319, N@10: 0.4432, M@10: 0.3842\n",
      "test H@20: 0.7243, N@20: 0.4666, M@20: 0.3906\n",
      "\n",
      "Epoch [260/500], Train Loss: 16.2593, Elapsed: train 0.50 test 1.72 *\n",
      "valid H@5: 0.5154, N@5: 0.3982, M@5: 0.3593\n",
      "valid H@10: 0.6220, N@10: 0.4328, M@10: 0.3737\n",
      "valid H@20: 0.7088, N@20: 0.4547, M@20: 0.3796\n",
      "\n",
      "test H@5: 0.5258, N@5: 0.4081, M@5: 0.3690\n",
      "test H@10: 0.6308, N@10: 0.4423, M@10: 0.3832\n",
      "test H@20: 0.7235, N@20: 0.4658, M@20: 0.3898\n",
      "\n",
      "Epoch [261/500], Train Loss: 16.3612, Elapsed: train 0.48 test 1.99 *\n",
      "valid H@5: 0.5110, N@5: 0.3978, M@5: 0.3602\n",
      "valid H@10: 0.6208, N@10: 0.4335, M@10: 0.3750\n",
      "valid H@20: 0.7109, N@20: 0.4562, M@20: 0.3812\n",
      "\n",
      "test H@5: 0.5273, N@5: 0.4091, M@5: 0.3698\n",
      "test H@10: 0.6269, N@10: 0.4417, M@10: 0.3835\n",
      "test H@20: 0.7231, N@20: 0.4662, M@20: 0.3903\n",
      "\n",
      "Epoch [262/500], Train Loss: 16.3103, Elapsed: train 0.48 test 1.93\n",
      "valid H@5: 0.5120, N@5: 0.3980, M@5: 0.3601\n",
      "valid H@10: 0.6189, N@10: 0.4328, M@10: 0.3746\n",
      "valid H@20: 0.7095, N@20: 0.4557, M@20: 0.3809\n",
      "\n",
      "test H@5: 0.5269, N@5: 0.4093, M@5: 0.3703\n",
      "test H@10: 0.6304, N@10: 0.4430, M@10: 0.3843\n",
      "test H@20: 0.7227, N@20: 0.4664, M@20: 0.3908\n",
      "\n",
      "Epoch [263/500], Train Loss: 16.3008, Elapsed: train 0.53 test 1.89\n",
      "valid H@5: 0.5150, N@5: 0.3991, M@5: 0.3607\n",
      "valid H@10: 0.6193, N@10: 0.4330, M@10: 0.3747\n",
      "valid H@20: 0.7101, N@20: 0.4559, M@20: 0.3810\n",
      "\n",
      "test H@5: 0.5267, N@5: 0.4091, M@5: 0.3701\n",
      "test H@10: 0.6323, N@10: 0.4435, M@10: 0.3844\n",
      "test H@20: 0.7229, N@20: 0.4665, M@20: 0.3908\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [264/500], Train Loss: 16.3850, Elapsed: train 0.45 test 1.93 *\n",
      "valid H@5: 0.5187, N@5: 0.4014, M@5: 0.3624\n",
      "valid H@10: 0.6195, N@10: 0.4341, M@10: 0.3760\n",
      "valid H@20: 0.7097, N@20: 0.4569, M@20: 0.3823\n",
      "\n",
      "test H@5: 0.5286, N@5: 0.4095, M@5: 0.3700\n",
      "test H@10: 0.6319, N@10: 0.4430, M@10: 0.3839\n",
      "test H@20: 0.7208, N@20: 0.4656, M@20: 0.3902\n",
      "\n",
      "Epoch [265/500], Train Loss: 16.2632, Elapsed: train 0.50 test 1.80\n",
      "valid H@5: 0.5170, N@5: 0.4010, M@5: 0.3625\n",
      "valid H@10: 0.6179, N@10: 0.4338, M@10: 0.3762\n",
      "valid H@20: 0.7099, N@20: 0.4572, M@20: 0.3826\n",
      "\n",
      "test H@5: 0.5263, N@5: 0.4086, M@5: 0.3695\n",
      "test H@10: 0.6310, N@10: 0.4426, M@10: 0.3836\n",
      "test H@20: 0.7214, N@20: 0.4657, M@20: 0.3901\n",
      "\n",
      "Epoch [266/500], Train Loss: 16.4641, Elapsed: train 0.47 test 1.80\n",
      "valid H@5: 0.5156, N@5: 0.4009, M@5: 0.3628\n",
      "valid H@10: 0.6187, N@10: 0.4345, M@10: 0.3768\n",
      "valid H@20: 0.7103, N@20: 0.4576, M@20: 0.3832\n",
      "\n",
      "test H@5: 0.5269, N@5: 0.4088, M@5: 0.3696\n",
      "test H@10: 0.6321, N@10: 0.4429, M@10: 0.3837\n",
      "test H@20: 0.7237, N@20: 0.4661, M@20: 0.3901\n",
      "\n",
      "Epoch [267/500], Train Loss: 16.2224, Elapsed: train 0.53 test 1.81\n",
      "valid H@5: 0.5154, N@5: 0.3999, M@5: 0.3616\n",
      "valid H@10: 0.6189, N@10: 0.4335, M@10: 0.3756\n",
      "valid H@20: 0.7089, N@20: 0.4563, M@20: 0.3818\n",
      "\n",
      "test H@5: 0.5279, N@5: 0.4081, M@5: 0.3683\n",
      "test H@10: 0.6333, N@10: 0.4421, M@10: 0.3824\n",
      "test H@20: 0.7229, N@20: 0.4649, M@20: 0.3886\n",
      "\n",
      "Epoch [268/500], Train Loss: 16.1566, Elapsed: train 0.45 test 1.91\n",
      "valid H@5: 0.5150, N@5: 0.3986, M@5: 0.3599\n",
      "valid H@10: 0.6183, N@10: 0.4322, M@10: 0.3739\n",
      "valid H@20: 0.7089, N@20: 0.4552, M@20: 0.3803\n",
      "\n",
      "test H@5: 0.5267, N@5: 0.4087, M@5: 0.3696\n",
      "test H@10: 0.6321, N@10: 0.4429, M@10: 0.3837\n",
      "test H@20: 0.7231, N@20: 0.4660, M@20: 0.3901\n",
      "\n",
      "Epoch [269/500], Train Loss: 16.2283, Elapsed: train 0.50 test 1.81\n",
      "valid H@5: 0.5166, N@5: 0.3988, M@5: 0.3597\n",
      "valid H@10: 0.6174, N@10: 0.4315, M@10: 0.3733\n",
      "valid H@20: 0.7093, N@20: 0.4549, M@20: 0.3798\n",
      "\n",
      "test H@5: 0.5292, N@5: 0.4100, M@5: 0.3704\n",
      "test H@10: 0.6336, N@10: 0.4438, M@10: 0.3844\n",
      "test H@20: 0.7235, N@20: 0.4665, M@20: 0.3906\n",
      "\n",
      "Epoch [270/500], Train Loss: 16.3003, Elapsed: train 0.54 test 1.78\n",
      "valid H@5: 0.5148, N@5: 0.3986, M@5: 0.3599\n",
      "valid H@10: 0.6166, N@10: 0.4317, M@10: 0.3737\n",
      "valid H@20: 0.7103, N@20: 0.4555, M@20: 0.3803\n",
      "\n",
      "test H@5: 0.5271, N@5: 0.4091, M@5: 0.3700\n",
      "test H@10: 0.6340, N@10: 0.4438, M@10: 0.3843\n",
      "test H@20: 0.7208, N@20: 0.4658, M@20: 0.3904\n",
      "\n",
      "Epoch [271/500], Train Loss: 16.4516, Elapsed: train 0.47 test 1.77\n",
      "valid H@5: 0.5168, N@5: 0.3987, M@5: 0.3595\n",
      "valid H@10: 0.6185, N@10: 0.4317, M@10: 0.3731\n",
      "valid H@20: 0.7088, N@20: 0.4546, M@20: 0.3795\n",
      "\n",
      "test H@5: 0.5296, N@5: 0.4097, M@5: 0.3700\n",
      "test H@10: 0.6340, N@10: 0.4435, M@10: 0.3840\n",
      "test H@20: 0.7212, N@20: 0.4657, M@20: 0.3901\n",
      "\n",
      "Epoch [272/500], Train Loss: 16.1479, Elapsed: train 0.47 test 1.80\n",
      "valid H@5: 0.5148, N@5: 0.3989, M@5: 0.3604\n",
      "valid H@10: 0.6191, N@10: 0.4327, M@10: 0.3744\n",
      "valid H@20: 0.7074, N@20: 0.4552, M@20: 0.3806\n",
      "\n",
      "test H@5: 0.5275, N@5: 0.4108, M@5: 0.3721\n",
      "test H@10: 0.6333, N@10: 0.4450, M@10: 0.3862\n",
      "test H@20: 0.7214, N@20: 0.4674, M@20: 0.3924\n",
      "\n",
      "Epoch [273/500], Train Loss: 16.1876, Elapsed: train 0.48 test 1.81\n",
      "valid H@5: 0.5147, N@5: 0.3990, M@5: 0.3606\n",
      "valid H@10: 0.6174, N@10: 0.4323, M@10: 0.3744\n",
      "valid H@20: 0.7086, N@20: 0.4555, M@20: 0.3808\n",
      "\n",
      "test H@5: 0.5290, N@5: 0.4115, M@5: 0.3724\n",
      "test H@10: 0.6304, N@10: 0.4443, M@10: 0.3860\n",
      "test H@20: 0.7212, N@20: 0.4674, M@20: 0.3924\n",
      "\n",
      "Epoch [274/500], Train Loss: 15.9475, Elapsed: train 0.51 test 1.77\n",
      "valid H@5: 0.5133, N@5: 0.3986, M@5: 0.3605\n",
      "valid H@10: 0.6191, N@10: 0.4329, M@10: 0.3747\n",
      "valid H@20: 0.7086, N@20: 0.4556, M@20: 0.3810\n",
      "\n",
      "test H@5: 0.5308, N@5: 0.4113, M@5: 0.3716\n",
      "test H@10: 0.6323, N@10: 0.4441, M@10: 0.3851\n",
      "test H@20: 0.7229, N@20: 0.4672, M@20: 0.3916\n",
      "\n",
      "Epoch [275/500], Train Loss: 16.1617, Elapsed: train 0.48 test 1.78\n",
      "valid H@5: 0.5160, N@5: 0.3999, M@5: 0.3614\n",
      "valid H@10: 0.6187, N@10: 0.4332, M@10: 0.3752\n",
      "valid H@20: 0.7078, N@20: 0.4558, M@20: 0.3814\n",
      "\n",
      "test H@5: 0.5309, N@5: 0.4119, M@5: 0.3724\n",
      "test H@10: 0.6321, N@10: 0.4447, M@10: 0.3860\n",
      "test H@20: 0.7218, N@20: 0.4675, M@20: 0.3923\n",
      "\n",
      "Epoch [276/500], Train Loss: 16.1425, Elapsed: train 0.48 test 1.72\n",
      "valid H@5: 0.5127, N@5: 0.3982, M@5: 0.3601\n",
      "valid H@10: 0.6174, N@10: 0.4324, M@10: 0.3744\n",
      "valid H@20: 0.7089, N@20: 0.4556, M@20: 0.3808\n",
      "\n",
      "test H@5: 0.5296, N@5: 0.4108, M@5: 0.3714\n",
      "test H@10: 0.6315, N@10: 0.4437, M@10: 0.3850\n",
      "test H@20: 0.7222, N@20: 0.4668, M@20: 0.3914\n",
      "\n",
      "Epoch [277/500], Train Loss: 16.0941, Elapsed: train 0.57 test 2.15\n",
      "valid H@5: 0.5133, N@5: 0.3991, M@5: 0.3611\n",
      "valid H@10: 0.6154, N@10: 0.4324, M@10: 0.3750\n",
      "valid H@20: 0.7101, N@20: 0.4564, M@20: 0.3816\n",
      "\n",
      "test H@5: 0.5296, N@5: 0.4114, M@5: 0.3722\n",
      "test H@10: 0.6308, N@10: 0.4442, M@10: 0.3858\n",
      "test H@20: 0.7233, N@20: 0.4677, M@20: 0.3923\n",
      "\n",
      "Epoch [278/500], Train Loss: 16.1005, Elapsed: train 0.47 test 1.83\n",
      "valid H@5: 0.5148, N@5: 0.3990, M@5: 0.3606\n",
      "valid H@10: 0.6193, N@10: 0.4329, M@10: 0.3746\n",
      "valid H@20: 0.7095, N@20: 0.4557, M@20: 0.3809\n",
      "\n",
      "test H@5: 0.5308, N@5: 0.4115, M@5: 0.3720\n",
      "test H@10: 0.6325, N@10: 0.4444, M@10: 0.3855\n",
      "test H@20: 0.7256, N@20: 0.4680, M@20: 0.3921\n",
      "\n",
      "Epoch [279/500], Train Loss: 16.2698, Elapsed: train 0.45 test 1.78\n",
      "valid H@5: 0.5152, N@5: 0.3988, M@5: 0.3600\n",
      "valid H@10: 0.6170, N@10: 0.4319, M@10: 0.3738\n",
      "valid H@20: 0.7078, N@20: 0.4550, M@20: 0.3803\n",
      "\n",
      "test H@5: 0.5283, N@5: 0.4104, M@5: 0.3713\n",
      "test H@10: 0.6321, N@10: 0.4441, M@10: 0.3852\n",
      "test H@20: 0.7222, N@20: 0.4670, M@20: 0.3917\n",
      "\n",
      "Epoch [280/500], Train Loss: 15.9768, Elapsed: train 0.50 test 1.74\n",
      "valid H@5: 0.5166, N@5: 0.3997, M@5: 0.3609\n",
      "valid H@10: 0.6164, N@10: 0.4322, M@10: 0.3745\n",
      "valid H@20: 0.7105, N@20: 0.4561, M@20: 0.3810\n",
      "\n",
      "test H@5: 0.5281, N@5: 0.4110, M@5: 0.3722\n",
      "test H@10: 0.6333, N@10: 0.4451, M@10: 0.3863\n",
      "test H@20: 0.7247, N@20: 0.4683, M@20: 0.3927\n",
      "\n",
      "Epoch [281/500], Train Loss: 16.0414, Elapsed: train 0.49 test 1.90\n",
      "valid H@5: 0.5139, N@5: 0.3982, M@5: 0.3598\n",
      "valid H@10: 0.6164, N@10: 0.4316, M@10: 0.3737\n",
      "valid H@20: 0.7089, N@20: 0.4550, M@20: 0.3801\n",
      "\n",
      "test H@5: 0.5294, N@5: 0.4103, M@5: 0.3708\n",
      "test H@10: 0.6331, N@10: 0.4439, M@10: 0.3848\n",
      "test H@20: 0.7241, N@20: 0.4671, M@20: 0.3912\n",
      "\n",
      "Epoch [282/500], Train Loss: 16.1730, Elapsed: train 0.49 test 1.80\n",
      "valid H@5: 0.5139, N@5: 0.3983, M@5: 0.3599\n",
      "valid H@10: 0.6181, N@10: 0.4321, M@10: 0.3739\n",
      "valid H@20: 0.7095, N@20: 0.4553, M@20: 0.3803\n",
      "\n",
      "test H@5: 0.5285, N@5: 0.4094, M@5: 0.3700\n",
      "test H@10: 0.6354, N@10: 0.4440, M@10: 0.3843\n",
      "test H@20: 0.7233, N@20: 0.4664, M@20: 0.3905\n",
      "\n",
      "Epoch [283/500], Train Loss: 16.1234, Elapsed: train 0.48 test 1.89\n",
      "valid H@5: 0.5124, N@5: 0.3966, M@5: 0.3581\n",
      "valid H@10: 0.6174, N@10: 0.4309, M@10: 0.3724\n",
      "valid H@20: 0.7074, N@20: 0.4538, M@20: 0.3788\n",
      "\n",
      "test H@5: 0.5300, N@5: 0.4099, M@5: 0.3701\n",
      "test H@10: 0.6340, N@10: 0.4436, M@10: 0.3841\n",
      "test H@20: 0.7249, N@20: 0.4667, M@20: 0.3905\n",
      "\n",
      "Epoch [284/500], Train Loss: 16.2970, Elapsed: train 0.53 test 1.80\n",
      "valid H@5: 0.5147, N@5: 0.3972, M@5: 0.3582\n",
      "valid H@10: 0.6166, N@10: 0.4303, M@10: 0.3720\n",
      "valid H@20: 0.7084, N@20: 0.4536, M@20: 0.3784\n",
      "\n",
      "test H@5: 0.5286, N@5: 0.4093, M@5: 0.3696\n",
      "test H@10: 0.6350, N@10: 0.4436, M@10: 0.3838\n",
      "test H@20: 0.7243, N@20: 0.4663, M@20: 0.3901\n",
      "\n",
      "Epoch [285/500], Train Loss: 16.0189, Elapsed: train 0.53 test 2.08\n",
      "valid H@5: 0.5150, N@5: 0.3969, M@5: 0.3577\n",
      "valid H@10: 0.6170, N@10: 0.4301, M@10: 0.3715\n",
      "valid H@20: 0.7091, N@20: 0.4535, M@20: 0.3780\n",
      "\n",
      "test H@5: 0.5294, N@5: 0.4100, M@5: 0.3704\n",
      "test H@10: 0.6354, N@10: 0.4443, M@10: 0.3845\n",
      "test H@20: 0.7252, N@20: 0.4670, M@20: 0.3908\n",
      "\n",
      "Epoch [286/500], Train Loss: 16.1654, Elapsed: train 0.46 test 1.84\n",
      "valid H@5: 0.5139, N@5: 0.3966, M@5: 0.3576\n",
      "valid H@10: 0.6170, N@10: 0.4300, M@10: 0.3715\n",
      "valid H@20: 0.7072, N@20: 0.4530, M@20: 0.3779\n",
      "\n",
      "test H@5: 0.5283, N@5: 0.4097, M@5: 0.3704\n",
      "test H@10: 0.6327, N@10: 0.4436, M@10: 0.3844\n",
      "test H@20: 0.7226, N@20: 0.4665, M@20: 0.3908\n",
      "\n",
      "Epoch [287/500], Train Loss: 16.2148, Elapsed: train 0.55 test 1.79\n",
      "valid H@5: 0.5154, N@5: 0.3980, M@5: 0.3591\n",
      "valid H@10: 0.6114, N@10: 0.4293, M@10: 0.3721\n",
      "valid H@20: 0.7091, N@20: 0.4542, M@20: 0.3791\n",
      "\n",
      "test H@5: 0.5296, N@5: 0.4105, M@5: 0.3710\n",
      "test H@10: 0.6333, N@10: 0.4441, M@10: 0.3849\n",
      "test H@20: 0.7247, N@20: 0.4674, M@20: 0.3914\n",
      "\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Epoch [288/500], Train Loss: 16.1825, Elapsed: train 0.45 test 1.81\n",
      "valid H@5: 0.5141, N@5: 0.3972, M@5: 0.3584\n",
      "valid H@10: 0.6154, N@10: 0.4302, M@10: 0.3722\n",
      "valid H@20: 0.7089, N@20: 0.4539, M@20: 0.3787\n",
      "\n",
      "test H@5: 0.5275, N@5: 0.4104, M@5: 0.3715\n",
      "test H@10: 0.6329, N@10: 0.4447, M@10: 0.3858\n",
      "test H@20: 0.7256, N@20: 0.4682, M@20: 0.3923\n",
      "\n",
      "Epoch [289/500], Train Loss: 16.1864, Elapsed: train 0.47 test 1.79\n",
      "valid H@5: 0.5108, N@5: 0.3953, M@5: 0.3570\n",
      "valid H@10: 0.6156, N@10: 0.4296, M@10: 0.3713\n",
      "valid H@20: 0.7082, N@20: 0.4530, M@20: 0.3778\n",
      "\n",
      "test H@5: 0.5304, N@5: 0.4112, M@5: 0.3716\n",
      "test H@10: 0.6338, N@10: 0.4448, M@10: 0.3856\n",
      "test H@20: 0.7249, N@20: 0.4679, M@20: 0.3920\n",
      "\n",
      "Epoch [290/500], Train Loss: 15.8208, Elapsed: train 0.57 test 1.85\n",
      "valid H@5: 0.5145, N@5: 0.3975, M@5: 0.3587\n",
      "valid H@10: 0.6139, N@10: 0.4299, M@10: 0.3722\n",
      "valid H@20: 0.7082, N@20: 0.4539, M@20: 0.3789\n",
      "\n",
      "test H@5: 0.5300, N@5: 0.4109, M@5: 0.3714\n",
      "test H@10: 0.6312, N@10: 0.4438, M@10: 0.3851\n",
      "test H@20: 0.7268, N@20: 0.4681, M@20: 0.3918\n",
      "\n",
      "Epoch [291/500], Train Loss: 16.1307, Elapsed: train 0.51 test 1.88\n",
      "valid H@5: 0.5162, N@5: 0.3979, M@5: 0.3587\n",
      "valid H@10: 0.6151, N@10: 0.4301, M@10: 0.3721\n",
      "valid H@20: 0.7086, N@20: 0.4538, M@20: 0.3786\n",
      "\n",
      "test H@5: 0.5288, N@5: 0.4108, M@5: 0.3717\n",
      "test H@10: 0.6317, N@10: 0.4443, M@10: 0.3856\n",
      "test H@20: 0.7252, N@20: 0.4681, M@20: 0.3922\n",
      "\n",
      "Epoch [292/500], Train Loss: 16.3272, Elapsed: train 0.49 test 1.76\n",
      "valid H@5: 0.5152, N@5: 0.3971, M@5: 0.3580\n",
      "valid H@10: 0.6158, N@10: 0.4298, M@10: 0.3715\n",
      "valid H@20: 0.7091, N@20: 0.4534, M@20: 0.3780\n",
      "\n",
      "test H@5: 0.5275, N@5: 0.4114, M@5: 0.3728\n",
      "test H@10: 0.6335, N@10: 0.4458, M@10: 0.3871\n",
      "test H@20: 0.7249, N@20: 0.4690, M@20: 0.3935\n",
      "\n",
      "Epoch [293/500], Train Loss: 16.1704, Elapsed: train 0.54 test 1.65\n",
      "valid H@5: 0.5173, N@5: 0.3991, M@5: 0.3599\n",
      "valid H@10: 0.6162, N@10: 0.4312, M@10: 0.3732\n",
      "valid H@20: 0.7047, N@20: 0.4537, M@20: 0.3795\n",
      "\n",
      "test H@5: 0.5256, N@5: 0.4090, M@5: 0.3702\n",
      "test H@10: 0.6325, N@10: 0.4438, M@10: 0.3847\n",
      "test H@20: 0.7268, N@20: 0.4677, M@20: 0.3913\n",
      "\n",
      "Epoch [294/500], Train Loss: 16.0985, Elapsed: train 0.49 test 1.81\n",
      "valid H@5: 0.5171, N@5: 0.3986, M@5: 0.3593\n",
      "valid H@10: 0.6168, N@10: 0.4309, M@10: 0.3727\n",
      "valid H@20: 0.7057, N@20: 0.4536, M@20: 0.3790\n",
      "\n",
      "test H@5: 0.5292, N@5: 0.4107, M@5: 0.3713\n",
      "test H@10: 0.6333, N@10: 0.4445, M@10: 0.3853\n",
      "test H@20: 0.7262, N@20: 0.4680, M@20: 0.3918\n",
      "\n",
      "BEST EPOCH:: 264\n",
      "valid\n",
      "{'H05': 0.5187, 'M05': 0.3624, 'N05': 0.4014, 'H10': 0.622, 'M10': 0.3737, 'N10': 0.4328, 'H20': 0.7109, 'M20': 0.3812, 'N20': 0.4562}\n",
      "test\n",
      "{'H05': 0.5286, 'M05': 0.37, 'N05': 0.4095, 'H10': 0.6308, 'M10': 0.3832, 'N10': 0.4423, 'H20': 0.7231, 'M20': 0.3903, 'N20': 0.4662}\n"
     ]
    }
   ],
   "source": [
    "# start train\n",
    "LOO_run(opt(max_epoch, early_stop, es_epoch), model, gpu, optimizer, train_loader, test_dataset, model_save_path=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "torch",
   "language": "python",
   "name": "torch"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}