notebooks/rs-check-how-works.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "39b38516-c6d4-4d7c-8338-98e65bab968c",
   "metadata": {
    "tags": []
   },
   "source": [
    "### References\n",
    "\n",
    "* [DeepFM: A Factorization-Machine based Neural Network for CTR Prediction](https://arxiv.org/pdf/1703.04247.pdf)\n",
    "* [Factorization Machines](https://d2l.ai/chapter_recommender-systems/fm.html)\n",
    "* [Sistemas de Recomendación (Parte 1): Filtros Colaborativos | Clase 22 | Aprendizaje Profundo 2021](https://www.youtube.com/watch?v=YAvX3BBh7U4)\n",
    "* https://github.com/rixwew/pytorch-fm\n",
    "* https://www.kaggle.com/c/avazu-ctr-prediction/data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "d2f24257-3bd5-490f-949a-54c0fb32f862",
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys\n",
    "sys.path.append('../src')\n",
    "\n",
    "import logging\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "import torch\n",
    "import torch.nn.functional as F\n",
    "from torch.utils.data import DataLoader, random_split\n",
    "\n",
    "from pytorch_common.util import get_device, set_device_name \n",
    "\n",
    "from dataset.movielens import MovieLens1MDataset, MovieLens20MDataset\n",
    "\n",
    "from sklearn.metrics import roc_auc_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "49dd99cd-29cd-4f77-9911-86b41f734ca9",
   "metadata": {},
   "outputs": [],
   "source": [
    "set_device_name('gpu')\n",
    "\n",
    "device = get_device()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "7a6a4303-2c13-48ed-8012-3c36b9dd52b5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_dataset(name):\n",
    "    if '1m' == name:\n",
    "        dataset_path = '../datasets/ml-1m/ratings.dat'\n",
    "        dataset = MovieLens1MDataset(dataset_path=dataset_path)\n",
    "    else:\n",
    "        dataset_path = '../datasets/ml-20m/ratings.csv'\n",
    "        dataset = MovieLens20MDataset(dataset_path=dataset_path)\n",
    "\n",
    "    logging.info('{} dataset loaded! Shape: {}'.format(dataset_path, dataset.shape))\n",
    "    logging.info('Target count: {}'.format(dataset.targets_count()))\n",
    "\n",
    "    return dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "c585621c-6069-4d17-9fe8-094ff8fd051f",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataset = load_dataset('1m')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "0adc3666-de7e-4276-8b32-f6c613246432",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000209"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "4b95478b-cf79-49d4-9047-8cdb52584563",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{0.0: 424928, 1.0: 575281}"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.targets_count()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "30a675d0-ee37-4f68-a416-8b52606d9aa7",
   "metadata": {},
   "source": [
    "(UserID, MovieID) pairs:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "06fd24fc-fae6-4e36-982d-9ac19a17146d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000209, 2)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.items.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "7ce713f2-e05a-404f-bd7b-2976f2c6c114",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[   0, 1192],\n",
       "       [   0,  660],\n",
       "       [   0,  913],\n",
       "       [   0, 3407],\n",
       "       [   0, 2354]])"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.items[0:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "d46d57e5-94ef-4bc1-a18b-f66ba616db99",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "6040"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset.user_ids())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "ee934b85-dc8a-420c-ba73-19dece331c72",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3706"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(dataset.movie_ids())"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "74318477-e029-4e18-b40a-c0b6ae34f672",
   "metadata": {},
   "source": [
    "Max UserID/MovieID:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "a4e99383-0836-4aae-ae08-de9caf31fb4b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([6040, 3952])"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset.field_dims"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c8390172-04af-4fc3-862c-0ba3f4967d4a",
   "metadata": {},
   "source": [
    "FeaturesEmbedding:\n",
    "\n",
    "Este modulo es una capa de embedding  la cual puedo alojar los embedding de N features en una sola lookup table. Por esta cuestion se guarla en office para cada feature y tambien es la razon por la cual al dimencion de la lookup table se un array con las dimenciones de cada feature. Ej: UserID tiene 100 ids pero ModieOd 1000 entonces field_dims == [100, 1000]. Luego para hacer el forward se lse pasa como input un array de lotes(batches) donde cada observacion es un array de features. ej.: [UserID, MovieID]."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "18eb426c-91d8-4548-b8a5-197d4eb532f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "class ModelMixin(object):\n",
    "    @property\n",
    "    def weights(self): return list(self.named_parameters())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "357cbc7a-de79-40dd-9b96-9bb757b28771",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeaturesEmbedding(torch.nn.Module, ModelMixin):\n",
    "\n",
    "    def __init__(self, emb_lookup_table_size, emb_vector_size):\n",
    "        super().__init__()\n",
    "        self.embedding = torch.nn.Embedding(\n",
    "            sum(emb_lookup_table_size), \n",
    "            emb_vector_size\n",
    "        )\n",
    "            self.offsets = np.array(\n",
    "                (0, *np.cumsum(emb_lookup_table_size)[:-1]), \n",
    "                dtype=np.long\n",
    "            )\n",
    "        torch.nn.init.xavier_uniform_(self.embedding.weight.data)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        :param x: Long tensor of size ``(batch_size, num_fields)``\n",
    "        \"\"\"\n",
    "            x = x + x.new_tensor(self.offsets).unsqueeze(0)\n",
    "        return self.embedding(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "9cd62bff-c1e7-4072-8933-7d87462d5ba9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'embedding.weight'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1039</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7616</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1775</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0746</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3711</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2336</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2377</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0135</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0593</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2405</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.4634</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7007</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5115</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2735</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3944</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2158</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5677</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3922</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7110</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6970</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3081</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'embedding.weight'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.1039\u001b[0m, \u001b[1;36m-0.7616\u001b[0m, \u001b[1;36m-0.1775\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.0746\u001b[0m,  \u001b[1;36m0.3711\u001b[0m,  \u001b[1;36m0.2336\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.2377\u001b[0m, \u001b[1;36m-0.0135\u001b[0m,  \u001b[1;36m0.0593\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.2405\u001b[0m,  \u001b[1;36m0.4634\u001b[0m, \u001b[1;36m-0.7007\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.5115\u001b[0m, \u001b[1;36m-0.2735\u001b[0m, \u001b[1;36m-0.3944\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.2158\u001b[0m,  \u001b[1;36m0.5677\u001b[0m, \u001b[1;36m-0.3922\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.7110\u001b[0m,  \u001b[1;36m0.6970\u001b[0m,  \u001b[1;36m0.3081\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emb_lookup_table_size = [3, 4]\n",
    "emb_vector_size = 3\n",
    "emb = FeaturesEmbedding(emb_lookup_table_size, emb_vector_size)\n",
    "emb.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "06d0084d-93a9-4571-992b-b80aa346beb5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m2\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x = torch.LongTensor([\n",
    "    [0, 1], \n",
    "    [2, 3]\n",
    "])\n",
    "x.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "f1deac38-e0a4-4222-ac4d-6702edf7c539",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">array</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35marray\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "offsets = np.array((0, *np.cumsum(emb_lookup_table_size)[:-1]), dtype=np.long)\n",
    "offsets"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "9e79e79b-824b-4c95-bbc7-e27e56434fdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">]])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "x.new_tensor(offsets).unsqueeze(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "45fe1e65-8ad3-4c85-8885-d0046172006d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">]])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m, \u001b[1;36m4\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m2\u001b[0m, \u001b[1;36m6\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y = x + x.new_tensor(offsets).unsqueeze(0)\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "761aeb95-abd7-486d-be66-c7ed9b8ff78f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m2\u001b[0m, \u001b[1;36m2\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "1bc7a144-182a-4243-943f-f9e74ba155c8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1039</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7616</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1775</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5115</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2735</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3944</span><span style=\"font-weight: bold\">]]</span>,\n",
       "\n",
       "        <span style=\"font-weight: bold\">[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2377</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0135</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0593</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7110</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6970</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3081</span><span style=\"font-weight: bold\">]]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">EmbeddingBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.1039\u001b[0m, \u001b[1;36m-0.7616\u001b[0m, \u001b[1;36m-0.1775\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m\u001b[1;36m-0.5115\u001b[0m, \u001b[1;36m-0.2735\u001b[0m, \u001b[1;36m-0.3944\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\n",
       "        \u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.2377\u001b[0m, \u001b[1;36m-0.0135\u001b[0m,  \u001b[1;36m0.0593\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m \u001b[1;36m0.7110\u001b[0m,  \u001b[1;36m0.6970\u001b[0m,  \u001b[1;36m0.3081\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mEmbeddingBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emb_out = emb(x)\n",
    "emb_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "0aafa47e-0b34-4c14-8778-b571fda9abca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m2\u001b[0m, \u001b[1;36m2\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emb_out.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "661c50a1-2122-4ed4-9ad8-c7e12ce4b871",
   "metadata": {},
   "source": [
    "La salida de la capa embedding es la concatenacion de los vectores de embedding correspondientes a la cantidad de features para la cual se creo el la capa FeaturesEmbedding. En este caso tenemso dos features UserID y MovieId, por lo cual solo teneso dos tablas de embedding (almbas dentro de la misma lookup table). "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "7153d313-d98f-4954-96d3-192d82f9de47",
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_output = len(emb_lookup_table_size) * emb_vector_size"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "2f8aea6b-2206-4af2-b006-2c585a9f175a",
   "metadata": {},
   "source": [
    "Cada embedding tiene 3 posiciones y tenemos dos features, por eso la salida tiene dimencion 6."
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b8055c4c-1ef4-47d5-93f5-13cee242c4f8",
   "metadata": {},
   "source": [
    "### FeaturesLinear"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "e84c1109-73ff-4f3f-b08f-c044282705ed",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FeaturesLinear(torch.nn.Module, ModelMixin):\n",
    "    def __init__(self, emb_lookup_table_size, output_dim=1):\n",
    "        super().__init__()\n",
    "        self.fc      = FeaturesEmbedding(emb_lookup_table_size, output_dim)\n",
    "        self.bias    = torch.nn.Parameter(torch.zeros((output_dim,)))\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        :param x: Long tensor of size ``(batch_size, num_fields)``\n",
    "        \"\"\"\n",
    "        return torch.sum(self.fc(x), dim=1) + self.bias\n",
    "\n",
    "    @property\n",
    "        def weight(self):\n",
    "        return self.fc.weight, self.bias"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "ba78dcc5-49f9-48ba-893e-950aaba0bb71",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FeaturesLinear</span><span style=\"font-weight: bold\">(</span>\n",
       "  <span style=\"font-weight: bold\">(</span>fc<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FeaturesEmbedding</span><span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">(</span>embedding<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Embedding</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mFeaturesLinear\u001b[0m\u001b[1m(\u001b[0m\n",
       "  \u001b[1m(\u001b[0mfc\u001b[1m)\u001b[0m: \u001b[1;35mFeaturesEmbedding\u001b[0m\u001b[1m(\u001b[0m\n",
       "    \u001b[1m(\u001b[0membedding\u001b[1m)\u001b[0m: \u001b[1;35mEmbedding\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m7\u001b[0m, \u001b[1;36m1\u001b[0m\u001b[1m)\u001b[0m\n",
       "  \u001b[1m)\u001b[0m\n",
       "\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fl = FeaturesLinear(emb_lookup_table_size)\n",
    "fl"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "992e45cb-7ace-4d69-b3b1-64cb962e3556",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008000; text-decoration-color: #008000\">'bias'</span>, Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>.<span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">))</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'fc.embedding.weight'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8050</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.4710</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7969</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.4270</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0210</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.5215</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6105</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[32m'bias'\u001b[0m, Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m.\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'fc.embedding.weight'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.8050\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.4710\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.7969\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.4270\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.0210\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.5215\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.6105\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fl.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "3d9308f8-b10c-47de-9297-383795231c61",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8050</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0210</span><span style=\"font-weight: bold\">]]</span>,\n",
       "\n",
       "        <span style=\"font-weight: bold\">[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7969</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6105</span><span style=\"font-weight: bold\">]]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">EmbeddingBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.8050\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m\u001b[1;36m-0.0210\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\n",
       "        \u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.7969\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m \u001b[1;36m0.6105\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mEmbeddingBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fl.fc(x)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "920ea3f1-b6a2-4e6f-9ad1-0adcf5f118da",
   "metadata": {},
   "source": [
    "### MultiLayerPerceptron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "c09459b5-5d41-4b61-bfc5-c4af4b99b478",
   "metadata": {},
   "outputs": [],
   "source": [
    "class MultiLayerPerceptron(torch.nn.Module, ModelMixin):\n",
    "\n",
    "    def __init__(self, input_units, units_per_layer, dropout, output_layer=True):\n",
    "        super().__init__()\n",
    "        layers = list()\n",
    "\n",
    "        for units in units_per_layer:\n",
    "            layers.append(torch.nn.Linear(input_units, units))\n",
    "            layers.append(torch.nn.BatchNorm1d(units))\n",
    "            layers.append(torch.nn.ReLU())\n",
    "            layers.append(torch.nn.Dropout(p=dropout))\n",
    "            input_units = units\n",
    "\n",
    "        if output_layer:\n",
    "            layers.append(torch.nn.Linear(input_units, 1))\n",
    "        \n",
    "        self.mlp = torch.nn.Sequential(*layers)\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        :param x: Float tensor of size ``(batch_size, embed_dim)``\n",
    "        \"\"\"\n",
    "        return self.mlp(x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3702b08f-2319-476f-8ef0-3459b90c3580",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">MultiLayerPerceptron</span><span style=\"font-weight: bold\">(</span>\n",
       "  <span style=\"font-weight: bold\">(</span>mlp<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Sequential</span><span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">BatchNorm1d</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #808000; text-decoration-color: #808000\">eps</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1e-05</span>, <span style=\"color: #808000; text-decoration-color: #808000\">momentum</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1</span>, <span style=\"color: #808000; text-decoration-color: #808000\">affine</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span>, <span style=\"color: #808000; text-decoration-color: #808000\">track_running_stats</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ReLU</span><span style=\"font-weight: bold\">()</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Dropout</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">p</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span>, <span style=\"color: #808000; text-decoration-color: #808000\">inplace</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">BatchNorm1d</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #808000; text-decoration-color: #808000\">eps</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1e-05</span>, <span style=\"color: #808000; text-decoration-color: #808000\">momentum</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1</span>, <span style=\"color: #808000; text-decoration-color: #808000\">affine</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span>, <span style=\"color: #808000; text-decoration-color: #808000\">track_running_stats</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ReLU</span><span style=\"font-weight: bold\">()</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Dropout</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">p</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span>, <span style=\"color: #808000; text-decoration-color: #808000\">inplace</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mMultiLayerPerceptron\u001b[0m\u001b[1m(\u001b[0m\n",
       "  \u001b[1m(\u001b[0mmlp\u001b[1m)\u001b[0m: \u001b[1;35mSequential\u001b[0m\u001b[1m(\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m6\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m5\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mBatchNorm1d\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m5\u001b[0m, \u001b[33meps\u001b[0m=\u001b[1;36m1e\u001b[0m\u001b[1;36m-05\u001b[0m, \u001b[33mmomentum\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.1\u001b[0m, \u001b[33maffine\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mtrack_running_stats\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mReLU\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mDropout\u001b[0m\u001b[1m(\u001b[0m\u001b[33mp\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.2\u001b[0m, \u001b[33minplace\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m5\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m5\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mBatchNorm1d\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m5\u001b[0m, \u001b[33meps\u001b[0m=\u001b[1;36m1e\u001b[0m\u001b[1;36m-05\u001b[0m, \u001b[33mmomentum\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.1\u001b[0m, \u001b[33maffine\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mtrack_running_stats\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mReLU\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m7\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mDropout\u001b[0m\u001b[1m(\u001b[0m\u001b[33mp\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.2\u001b[0m, \u001b[33minplace\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m5\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m1\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "  \u001b[1m)\u001b[0m\n",
       "\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mlp = MultiLayerPerceptron(\n",
    "    input_units     = embedding_output, \n",
    "    units_per_layer = [5, 5], \n",
    "    dropout         = 0.2\n",
    ")\n",
    "mlp"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "a1dbc422-09c7-4816-9031-42a6adeb2c35",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"font-weight: bold\">[</span>\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.0.weight'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1710</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2687</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1666</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3099</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3865</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0487</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3926</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2640</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2705</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1789</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3134</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0326</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1136</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2276</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2778</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0648</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3054</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2958</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3119</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0558</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3434</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3861</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3587</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0651</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3519</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1851</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1115</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2830</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2074</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3886</span><span style=\"font-weight: bold\">]]</span>,\n",
       "       <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.0.bias'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1424</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0856</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2915</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0020</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2742</span><span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.1.weight'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>.<span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.1.bias'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>.<span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.4.weight'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0188</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2911</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2195</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1496</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2808</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1413</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2096</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0463</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1850</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3060</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1140</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2813</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2134</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0815</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0261</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0313</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1885</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.4395</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0145</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2277</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2985</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0281</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0466</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2660</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0020</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.4.bias'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2038</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1012</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2259</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1036</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3009</span><span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.5.weight'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>.<span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.5.bias'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>., <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span>.<span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span>\n",
       "        <span style=\"color: #008000; text-decoration-color: #008000\">'mlp.8.weight'</span>,\n",
       "        Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3484</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1798</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0146</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3939</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0253</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>,\n",
       "    <span style=\"font-weight: bold\">(</span><span style=\"color: #008000; text-decoration-color: #008000\">'mlp.8.bias'</span>, Parameter containing:\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3918</span><span style=\"font-weight: bold\">]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">requires_grad</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">))</span>\n",
       "<span style=\"font-weight: bold\">]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1m[\u001b[0m\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.0.weight'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.1710\u001b[0m,  \u001b[1;36m0.2687\u001b[0m,  \u001b[1;36m0.1666\u001b[0m, \u001b[1;36m-0.3099\u001b[0m, \u001b[1;36m-0.3865\u001b[0m,  \u001b[1;36m0.0487\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.3926\u001b[0m, \u001b[1;36m-0.2640\u001b[0m,  \u001b[1;36m0.2705\u001b[0m,  \u001b[1;36m0.1789\u001b[0m,  \u001b[1;36m0.3134\u001b[0m, \u001b[1;36m-0.0326\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.1136\u001b[0m,  \u001b[1;36m0.2276\u001b[0m, \u001b[1;36m-0.2778\u001b[0m,  \u001b[1;36m0.0648\u001b[0m,  \u001b[1;36m0.3054\u001b[0m,  \u001b[1;36m0.2958\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.3119\u001b[0m,  \u001b[1;36m0.0558\u001b[0m,  \u001b[1;36m0.3434\u001b[0m, \u001b[1;36m-0.3861\u001b[0m, \u001b[1;36m-0.3587\u001b[0m,  \u001b[1;36m0.0651\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.3519\u001b[0m,  \u001b[1;36m0.1851\u001b[0m,  \u001b[1;36m0.1115\u001b[0m, \u001b[1;36m-0.2830\u001b[0m, \u001b[1;36m-0.2074\u001b[0m, \u001b[1;36m-0.3886\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m,\n",
       "       \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.0.bias'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.1424\u001b[0m, \u001b[1;36m-0.0856\u001b[0m, \u001b[1;36m-0.2915\u001b[0m,  \u001b[1;36m0.0020\u001b[0m, \u001b[1;36m-0.2742\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.1.weight'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m.\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.1.bias'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m.\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.4.weight'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.0188\u001b[0m, \u001b[1;36m-0.2911\u001b[0m, \u001b[1;36m-0.2195\u001b[0m, \u001b[1;36m-0.1496\u001b[0m,  \u001b[1;36m0.2808\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.1413\u001b[0m,  \u001b[1;36m0.2096\u001b[0m, \u001b[1;36m-0.0463\u001b[0m, \u001b[1;36m-0.1850\u001b[0m, \u001b[1;36m-0.3060\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.1140\u001b[0m, \u001b[1;36m-0.2813\u001b[0m,  \u001b[1;36m0.2134\u001b[0m, \u001b[1;36m-0.0815\u001b[0m,  \u001b[1;36m0.0261\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.0313\u001b[0m, \u001b[1;36m-0.1885\u001b[0m, \u001b[1;36m-0.4395\u001b[0m, \u001b[1;36m-0.0145\u001b[0m,  \u001b[1;36m0.2277\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.2985\u001b[0m, \u001b[1;36m-0.0281\u001b[0m,  \u001b[1;36m0.0466\u001b[0m, \u001b[1;36m-0.2660\u001b[0m,  \u001b[1;36m0.0020\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.4.bias'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.2038\u001b[0m, \u001b[1;36m-0.1012\u001b[0m, \u001b[1;36m-0.2259\u001b[0m, \u001b[1;36m-0.1036\u001b[0m,  \u001b[1;36m0.3009\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.5.weight'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m., \u001b[1;36m1\u001b[0m.\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.5.bias'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m., \u001b[1;36m0\u001b[0m.\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\n",
       "        \u001b[32m'mlp.8.weight'\u001b[0m,\n",
       "        Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.3484\u001b[0m,  \u001b[1;36m0.1798\u001b[0m,  \u001b[1;36m0.0146\u001b[0m,  \u001b[1;36m0.3939\u001b[0m, \u001b[1;36m-0.0253\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m,\n",
       "    \u001b[1m(\u001b[0m\u001b[32m'mlp.8.bias'\u001b[0m, Parameter containing:\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.3918\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mrequires_grad\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\u001b[1m)\u001b[0m\n",
       "\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mlp.weights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "d16dc33d-a129-4cc4-85b8-e732edc2a20c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1039</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7616</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1775</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5115</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2735</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3944</span><span style=\"font-weight: bold\">]]</span>,\n",
       "\n",
       "        <span style=\"font-weight: bold\">[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2377</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0135</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0593</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7110</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6970</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3081</span><span style=\"font-weight: bold\">]]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">EmbeddingBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.1039\u001b[0m, \u001b[1;36m-0.7616\u001b[0m, \u001b[1;36m-0.1775\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m\u001b[1;36m-0.5115\u001b[0m, \u001b[1;36m-0.2735\u001b[0m, \u001b[1;36m-0.3944\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\n",
       "        \u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.2377\u001b[0m, \u001b[1;36m-0.0135\u001b[0m,  \u001b[1;36m0.0593\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m \u001b[1;36m0.7110\u001b[0m,  \u001b[1;36m0.6970\u001b[0m,  \u001b[1;36m0.3081\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mEmbeddingBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emb_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "21619dde-c06a-4cd7-b447-cc94f151d1aa",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1039</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7616</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1775</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5115</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2735</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3944</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2377</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0135</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0593</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7110</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6970</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3081</span><span style=\"font-weight: bold\">]]</span>,\n",
       "       <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">ViewBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.1039\u001b[0m, \u001b[1;36m-0.7616\u001b[0m, \u001b[1;36m-0.1775\u001b[0m, \u001b[1;36m-0.5115\u001b[0m, \u001b[1;36m-0.2735\u001b[0m, \u001b[1;36m-0.3944\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.2377\u001b[0m, \u001b[1;36m-0.0135\u001b[0m,  \u001b[1;36m0.0593\u001b[0m,  \u001b[1;36m0.7110\u001b[0m,  \u001b[1;36m0.6970\u001b[0m,  \u001b[1;36m0.3081\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m,\n",
       "       \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mViewBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mlp_input = emb_out.view(-1, embedding_output)\n",
    "mlp_input"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "209209ca-cd0d-4c55-a5d7-fd5d6bc268eb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3348</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1488</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">AddmmBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.3348\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m-0.1488\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mAddmmBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "mlp_output = mlp(mlp_input)\n",
    "mlp_output"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b7033f2a-eb57-44d4-b622-672ead6147c5",
   "metadata": {},
   "source": [
    "### FactorizationMachine"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "4f48dc98-c04a-44a7-a433-18a838b8535f",
   "metadata": {},
   "outputs": [],
   "source": [
    "class FactorizationMachine(torch.nn.Module, ModelMixin):\n",
    "\n",
    "    def __init__(self, reduce_sum=True):\n",
    "        super().__init__()\n",
    "        self.reduce_sum = reduce_sum\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        :param x: Float tensor of size ``(batch_size, num_fields, embed_dim)``\n",
    "        \"\"\"\n",
    "        square_of_sum = torch.sum(x, dim=1) ** 2\n",
    "        sum_of_square = torch.sum(x ** 2, dim=1)\n",
    "        ix = square_of_sum - sum_of_square\n",
    "        if self.reduce_sum:\n",
    "            ix = torch.sum(ix, dim=1, keepdim=True)\n",
    "\n",
    "        return 0.5 * ix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "9830f479-79dc-4a95-81e8-05db5985faa9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1039</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.7616</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.1775</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5115</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.2735</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.3944</span><span style=\"font-weight: bold\">]]</span>,\n",
       "\n",
       "        <span style=\"font-weight: bold\">[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2377</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.0135</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.0593</span><span style=\"font-weight: bold\">]</span>,\n",
       "         <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7110</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6970</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3081</span><span style=\"font-weight: bold\">]]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">EmbeddingBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.1039\u001b[0m, \u001b[1;36m-0.7616\u001b[0m, \u001b[1;36m-0.1775\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m\u001b[1;36m-0.5115\u001b[0m, \u001b[1;36m-0.2735\u001b[0m, \u001b[1;36m-0.3944\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\n",
       "        \u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.2377\u001b[0m, \u001b[1;36m-0.0135\u001b[0m,  \u001b[1;36m0.0593\u001b[0m\u001b[1m]\u001b[0m,\n",
       "         \u001b[1m[\u001b[0m \u001b[1;36m0.7110\u001b[0m,  \u001b[1;36m0.6970\u001b[0m,  \u001b[1;36m0.3081\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mEmbeddingBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emb_out"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "cd5c9323-e9b3-435f-9586-ec18b956dbdd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m2\u001b[0m, \u001b[1;36m2\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "emb_out.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "a8ca965a-7ab7-4a9e-8fdf-c258c26362f0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.4076</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.0350</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5719</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9487</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6835</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3674</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">SumBackward1</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.4076\u001b[0m, \u001b[1;36m-1.0350\u001b[0m, \u001b[1;36m-0.5719\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m0.9487\u001b[0m,  \u001b[1;36m0.6835\u001b[0m,  \u001b[1;36m0.3674\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mSumBackward1\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(emb_out, dim=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "da92c62d-2a0e-40f0-921f-1e702f74cbb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m2\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(emb_out, dim=1).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "155fd64b-62f6-4fc3-bbce-33a97cfadcc2",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.4076</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.0350</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.5719</span><span style=\"font-weight: bold\">]]</span>,\n",
       "\n",
       "        <span style=\"font-weight: bold\">[[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.9487</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.6835</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.3674</span><span style=\"font-weight: bold\">]]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">SumBackward1</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.4076\u001b[0m, \u001b[1;36m-1.0350\u001b[0m, \u001b[1;36m-0.5719\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m,\n",
       "\n",
       "        \u001b[1m[\u001b[0m\u001b[1m[\u001b[0m \u001b[1;36m0.9487\u001b[0m,  \u001b[1;36m0.6835\u001b[0m,  \u001b[1;36m0.3674\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mSumBackward1\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(emb_out, dim=1, keepdim=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "85596890-f335-46a2-ae0d-b633a35edbf7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">torch.Size</span><span style=\"font-weight: bold\">([</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">])</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1;35mtorch.Size\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m2\u001b[0m, \u001b[1;36m1\u001b[0m, \u001b[1;36m3\u001b[0m\u001b[1m]\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(emb_out, dim=1, keepdim=True).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "01165efc-7dd7-4ff0-b788-7746d4db519f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-0.8152</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2.0701</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-1.1437</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.8973</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.3670</span>,  <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.7348</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">AddBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-0.8152\u001b[0m, \u001b[1;36m-2.0701\u001b[0m, \u001b[1;36m-1.1437\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m1.8973\u001b[0m,  \u001b[1;36m1.3670\u001b[0m,  \u001b[1;36m0.7348\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mAddBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "sum_embs = torch.sum(emb_out, dim=1) + torch.sum(emb_out, dim=1)\n",
    "sum_embs"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "88257e4f-4d00-4aa5-9268-b39ce2a1fa3a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-4.0291</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3.9991</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">SumBackward1</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-4.0291\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m3.9991\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mSumBackward1\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(sum_embs, dim=1, keepdim=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "ebf99122-e0a3-48d1-9263-ccdc68489e10",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">-2.0145</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span> <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1.9995</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">MulBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m-2.0145\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m \u001b[1;36m1.9995\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mMulBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "torch.sum(sum_embs, dim=1, keepdim=True) * 0.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "id": "c0768520-d149-46f0-a3a5-edff0202bc1b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">tensor</span><span style=\"font-weight: bold\">([[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2251</span><span style=\"font-weight: bold\">]</span>,\n",
       "        <span style=\"font-weight: bold\">[</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1779</span><span style=\"font-weight: bold\">]]</span>, <span style=\"color: #808000; text-decoration-color: #808000\">grad_fn</span>=<span style=\"font-weight: bold\">&lt;</span><span style=\"color: #ff00ff; text-decoration-color: #ff00ff; font-weight: bold\">MulBackward0</span><span style=\"font-weight: bold\">&gt;)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mtensor\u001b[0m\u001b[1m(\u001b[0m\u001b[1m[\u001b[0m\u001b[1m[\u001b[0m\u001b[1;36m0.2251\u001b[0m\u001b[1m]\u001b[0m,\n",
       "        \u001b[1m[\u001b[0m\u001b[1;36m0.1779\u001b[0m\u001b[1m]\u001b[0m\u001b[1m]\u001b[0m, \u001b[33mgrad_fn\u001b[0m=\u001b[1m<\u001b[0m\u001b[1;95mMulBackward0\u001b[0m\u001b[1m>\u001b[0m\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fm = FactorizationMachine(emb_lookup_table_size)\n",
    "fm(emb_out)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 57,
   "id": "b8fdf2c8-8899-4605-a039-0223d34c688f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">[]</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m[\u001b[0m\u001b[1m]\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fm.weights"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b588bd86-a457-406a-8c10-757230b9dd71",
   "metadata": {},
   "source": [
    "### DeepFactorizationMachineModel"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 70,
   "id": "f39eced1-2e0d-41c7-9681-e5e28a64999d",
   "metadata": {},
   "outputs": [],
   "source": [
    "class DeepFactorizationMachineModel(torch.nn.Module, ModelMixin):\n",
    "    \"\"\"\n",
    "    A pytorch implementation of DeepFM.\n",
    "    Reference:\n",
    "        H Guo, et al. DeepFM: A Factorization-Machine based Neural Network for CTR Prediction, 2017.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, emb_lookup_table_size, emb_vector_size, mlp_units_per_layer, mlp_dropout):\n",
    "        super().__init__()\n",
    "        self.linear= FeaturesLinear(emb_lookup_table_size)\n",
    "    \n",
    "        self.fm = FactorizationMachine(reduce_sum=True)\n",
    "\n",
    "        self.embedding       = FeaturesEmbedding(emb_lookup_table_size, emb_vector_size)\n",
    "        self.emb_output_dim  = len(emb_lookup_table_size) * emb_vector_size\n",
    "\n",
    "        self.mlp = MultiLayerPerceptron(\n",
    "            input_units     = self.emb_output_dim, \n",
    "            units_per_layer = mlp_units_per_layer,\n",
    "            dropout         = mlp_dropout\n",
    "        )\n",
    "\n",
    "    def forward(self, x):\n",
    "        \"\"\"\n",
    "        :param x: Long tensor of size ``(batch_size, num_fields)``\n",
    "        \"\"\"\n",
    "        embed_x = self.embedding(x)\n",
    "\n",
    "        x = self.linear(x) + \\\n",
    "            self.fm(embed_x) + \\\n",
    "            self.mlp(embed_x.view(-1, self.emb_output_dim))\n",
    "\n",
    "        return torch.sigmoid(x.squeeze(1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "004c2261-e3c4-4aed-9c28-a10ab62acba3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">\n",
       "<span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">DeepFactorizationMachineModel</span><span style=\"font-weight: bold\">(</span>\n",
       "  <span style=\"font-weight: bold\">(</span>linear<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FeaturesLinear</span><span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">(</span>fc<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FeaturesEmbedding</span><span style=\"font-weight: bold\">(</span>\n",
       "      <span style=\"font-weight: bold\">(</span>embedding<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Embedding</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9992</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">(</span>fm<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FactorizationMachine</span><span style=\"font-weight: bold\">()</span>\n",
       "  <span style=\"font-weight: bold\">(</span>embedding<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">FeaturesEmbedding</span><span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">(</span>embedding<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Embedding</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">9992</span>, <span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">50</span><span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">(</span>mlp<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">MultiLayerPerceptron</span><span style=\"font-weight: bold\">(</span>\n",
       "    <span style=\"font-weight: bold\">(</span>mlp<span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Sequential</span><span style=\"font-weight: bold\">(</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">BatchNorm1d</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>, <span style=\"color: #808000; text-decoration-color: #808000\">eps</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1e-05</span>, <span style=\"color: #808000; text-decoration-color: #808000\">momentum</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1</span>, <span style=\"color: #808000; text-decoration-color: #808000\">affine</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span>, <span style=\"color: #808000; text-decoration-color: #808000\">track_running_stats</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">2</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ReLU</span><span style=\"font-weight: bold\">()</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">3</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Dropout</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">p</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span>, <span style=\"color: #808000; text-decoration-color: #808000\">inplace</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">4</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">5</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">BatchNorm1d</span><span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>, <span style=\"color: #808000; text-decoration-color: #808000\">eps</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1e-05</span>, <span style=\"color: #808000; text-decoration-color: #808000\">momentum</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.1</span>, <span style=\"color: #808000; text-decoration-color: #808000\">affine</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span>, <span style=\"color: #808000; text-decoration-color: #808000\">track_running_stats</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">6</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">ReLU</span><span style=\"font-weight: bold\">()</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">7</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Dropout</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">p</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">0.2</span>, <span style=\"color: #808000; text-decoration-color: #808000\">inplace</span>=<span style=\"color: #ff0000; text-decoration-color: #ff0000; font-style: italic\">False</span><span style=\"font-weight: bold\">)</span>\n",
       "      <span style=\"font-weight: bold\">(</span><span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">8</span><span style=\"font-weight: bold\">)</span>: <span style=\"color: #800080; text-decoration-color: #800080; font-weight: bold\">Linear</span><span style=\"font-weight: bold\">(</span><span style=\"color: #808000; text-decoration-color: #808000\">in_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">100</span>, <span style=\"color: #808000; text-decoration-color: #808000\">out_features</span>=<span style=\"color: #008080; text-decoration-color: #008080; font-weight: bold\">1</span>, <span style=\"color: #808000; text-decoration-color: #808000\">bias</span>=<span style=\"color: #00ff00; text-decoration-color: #00ff00; font-style: italic\">True</span><span style=\"font-weight: bold\">)</span>\n",
       "    <span style=\"font-weight: bold\">)</span>\n",
       "  <span style=\"font-weight: bold\">)</span>\n",
       "<span style=\"font-weight: bold\">)</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\n",
       "\u001b[1;35mDeepFactorizationMachineModel\u001b[0m\u001b[1m(\u001b[0m\n",
       "  \u001b[1m(\u001b[0mlinear\u001b[1m)\u001b[0m: \u001b[1;35mFeaturesLinear\u001b[0m\u001b[1m(\u001b[0m\n",
       "    \u001b[1m(\u001b[0mfc\u001b[1m)\u001b[0m: \u001b[1;35mFeaturesEmbedding\u001b[0m\u001b[1m(\u001b[0m\n",
       "      \u001b[1m(\u001b[0membedding\u001b[1m)\u001b[0m: \u001b[1;35mEmbedding\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m9992\u001b[0m, \u001b[1;36m1\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "  \u001b[1m)\u001b[0m\n",
       "  \u001b[1m(\u001b[0mfm\u001b[1m)\u001b[0m: \u001b[1;35mFactorizationMachine\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "  \u001b[1m(\u001b[0membedding\u001b[1m)\u001b[0m: \u001b[1;35mFeaturesEmbedding\u001b[0m\u001b[1m(\u001b[0m\n",
       "    \u001b[1m(\u001b[0membedding\u001b[1m)\u001b[0m: \u001b[1;35mEmbedding\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m9992\u001b[0m, \u001b[1;36m50\u001b[0m\u001b[1m)\u001b[0m\n",
       "  \u001b[1m)\u001b[0m\n",
       "  \u001b[1m(\u001b[0mmlp\u001b[1m)\u001b[0m: \u001b[1;35mMultiLayerPerceptron\u001b[0m\u001b[1m(\u001b[0m\n",
       "    \u001b[1m(\u001b[0mmlp\u001b[1m)\u001b[0m: \u001b[1;35mSequential\u001b[0m\u001b[1m(\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m0\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m100\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m100\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m1\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mBatchNorm1d\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m100\u001b[0m, \u001b[33meps\u001b[0m=\u001b[1;36m1e\u001b[0m\u001b[1;36m-05\u001b[0m, \u001b[33mmomentum\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.1\u001b[0m, \u001b[33maffine\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mtrack_running_stats\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m2\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mReLU\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m3\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mDropout\u001b[0m\u001b[1m(\u001b[0m\u001b[33mp\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.2\u001b[0m, \u001b[33minplace\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m4\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m100\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m100\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m5\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mBatchNorm1d\u001b[0m\u001b[1m(\u001b[0m\u001b[1;36m100\u001b[0m, \u001b[33meps\u001b[0m=\u001b[1;36m1e\u001b[0m\u001b[1;36m-05\u001b[0m, \u001b[33mmomentum\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.1\u001b[0m, \u001b[33maffine\u001b[0m=\u001b[3;92mTrue\u001b[0m, \u001b[33mtrack_running_stats\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m6\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mReLU\u001b[0m\u001b[1m(\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m7\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mDropout\u001b[0m\u001b[1m(\u001b[0m\u001b[33mp\u001b[0m=\u001b[1;36m0\u001b[0m\u001b[1;36m.2\u001b[0m, \u001b[33minplace\u001b[0m=\u001b[3;91mFalse\u001b[0m\u001b[1m)\u001b[0m\n",
       "      \u001b[1m(\u001b[0m\u001b[1;36m8\u001b[0m\u001b[1m)\u001b[0m: \u001b[1;35mLinear\u001b[0m\u001b[1m(\u001b[0m\u001b[33min_features\u001b[0m=\u001b[1;36m100\u001b[0m, \u001b[33mout_features\u001b[0m=\u001b[1;36m1\u001b[0m, \u001b[33mbias\u001b[0m=\u001b[3;92mTrue\u001b[0m\u001b[1m)\u001b[0m\n",
       "    \u001b[1m)\u001b[0m\n",
       "  \u001b[1m)\u001b[0m\n",
       "\u001b[1m)\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dfm = DeepFactorizationMachineModel(\n",
    "    emb_lookup_table_size  = dataset.field_dims,\n",
    "    emb_vector_size        = 50,\n",
    "    mlp_units_per_layer    = [100, 100],\n",
    "    mlp_dropout            = 0.2\n",
    "\n",
    ").to(device)\n",
    "dfm"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "id": "d74cdd79-7c6c-4ec6-8d02-93974e2ad61c",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tqdm\n",
    "def train(model, optimizer, data_loader, criterion, device, log_interval=100):\n",
    "    model.train()\n",
    "    total_loss = 0\n",
    "    \n",
    "    tk0 = tqdm.tqdm(data_loader, smoothing=0, mininterval=1.0)\n",
    "    for i, (fields, target) in enumerate(tk0):\n",
    "        fields, target = fields.to(device), target.to(device)\n",
    "        y = model(fields)\n",
    "        \n",
    "        loss = criterion(y, target.float())\n",
    "        model.zero_grad()\n",
    "        loss.backward()\n",
    "        optimizer.step()\n",
    "        \n",
    "        total_loss += loss.item()\n",
    "        \n",
    "        if (i + 1) % log_interval == 0:\n",
    "            tk0.set_postfix(loss=total_loss / log_interval)\n",
    "            total_loss = 0\n",
    "\n",
    "\n",
    "def test(model, data_loader, device):\n",
    "    model.eval()\n",
    "    targets, predicts = list(), list()\n",
    "    with torch.no_grad():\n",
    "        for fields, target in tqdm.tqdm(data_loader, smoothing=0, mininterval=1.0):\n",
    "            fields, target = fields.to(device), target.to(device)\n",
    "            y = model(fields)\n",
    "            targets.extend(target.tolist())\n",
    "            predicts.extend(y.tolist())\n",
    "    return roc_auc_score(targets, predicts)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 112,
   "id": "bbbc9e2e-8a9f-4b88-8e1a-8be5f001fa0e",
   "metadata": {},
   "outputs": [],
   "source": [
    "class EarlyStopper(object):\n",
    "\n",
    "    def __init__(self, num_trials, save_path):\n",
    "        self.num_trials = num_trials\n",
    "        self.trial_counter = 0\n",
    "        self.best_accuracy = 0\n",
    "        self.save_path = save_path\n",
    "\n",
    "    def is_continuable(self, model, accuracy):\n",
    "        if accuracy > self.best_accuracy:\n",
    "            self.best_accuracy = accuracy\n",
    "            self.trial_counter = 0\n",
    "            torch.save(model, self.save_path)\n",
    "            return True\n",
    "        elif self.trial_counter + 1 < self.num_trials:\n",
    "            self.trial_counter += 1\n",
    "            return True\n",
    "        else:\n",
    "            return False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "id": "b25f9c35-28c0-4897-953f-dc631c38304b",
   "metadata": {},
   "outputs": [],
   "source": [
    "def train_val_test_split(X, train_percent=0.7, val_percent=0.15):\n",
    "    train_length = int(len(X) * train_percent)\n",
    "    valid_length = int(len(X) * val_percent)\n",
    "    test_length  = len(X) - train_length - valid_length\n",
    "\n",
    "    return random_split(X, (train_length, val_length, test_length))\n",
    "\n",
    "train_dataset, valid_dataset, test_dataset = train_val_test_split(dataset)\n",
    "\n",
    "batch_size   = 2048\n",
    "\n",
    "train_data_loader = DataLoader(train_dataset, batch_size=batch_size, num_workers=8)\n",
    "valid_data_loader = DataLoader(valid_dataset, batch_size=batch_size, num_workers=8)\n",
    "test_data_loader  = DataLoader(test_dataset,  batch_size=batch_size, num_workers=8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "4927717b-1da8-437e-bc6e-bfc3b7bcd0fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "learning_rate = 0.0000001\n",
    "weight_decay  = 1e-6\n",
    "epochs        = 500\n",
    "criterion = torch.nn.BCELoss()\n",
    "optimizer = torch.optim.Adam(\n",
    "    params = dfm.parameters(), \n",
    "    lr=learning_rate, \n",
    "    weight_decay=weight_decay\n",
    ")\n",
    "early_stopper = EarlyStopper(\n",
    "    num_trials=2, \n",
    "    save_path=f'./dfm.pt'\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "id": "44df31bb-d1f1-4c9b-8567-1dba52b5066b",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:01<00:00, 275.03it/s, loss=0.449]\n",
      "100%|██████████| 49/49 [00:00<00:00, 173.29it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 0 validation: auc: 0.8535864606658227\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:01<00:00, 286.66it/s, loss=0.449]\n",
      "100%|██████████| 49/49 [00:00<00:00, 162.42it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 1 validation: auc: 0.8535842706869562\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 391/391 [00:01<00:00, 284.69it/s, loss=0.449]\n",
      "100%|██████████| 49/49 [00:00<00:00, 153.67it/s]\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "epoch: 2 validation: auc: 0.8535806560971877\n",
      "validation: best auc: 0.8535864606658227\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 49/49 [00:00<00:00, 155.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "test auc: 0.8536817549862673\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "for epoch in range(epochs):\n",
    "    train(dfm, optimizer, train_data_loader, criterion, device)\n",
    "    auc = test(dfm, valid_data_loader, device)\n",
    "    print('epoch:', epoch, 'validation: auc:', auc)\n",
    "\n",
    "    if not early_stopper.is_continuable(dfm, auc):\n",
    "        print(f'validation: best auc: {early_stopper.best_accuracy}')\n",
    "        break\n",
    "\n",
    "auc = test(dfm, test_data_loader, device)\n",
    "\n",
    "print(f'test auc: {auc}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1bf07b8a-39f7-4ddf-888b-0e77b52a634b",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.7"
  },
  "widgets": {
   "application/vnd.jupyter.widget-state+json": {
    "state": {},
    "version_major": 2,
    "version_minor": 0
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}