sklearn_dataset_training.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "%pip install --upgrade poutyne\n",
    "%matplotlib inline\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np\n",
    "\n",
    "from sklearn.datasets import fetch_california_housing\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.svm import SVR\n",
    "from sklearn.metrics import mean_squared_error, r2_score\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "import torch.nn.functional as F\n",
    "import torch.optim as optim\n",
    "from torch.utils.data import TensorDataset, DataLoader\n",
    "\n",
    "from poutyne import Model, EpochProgressionCallback, BestModelRestore\n",
    "import torchmetrics\n",
    "\n",
    "plt.rcParams[\"figure.dpi\"] = 150"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some constants we will need."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "cuda_device = 0\n",
    "device = torch.device(\"cuda:%d\" % cuda_device if torch.cuda.is_available() else \"cpu\")\n",
    "\n",
    "batch_size = 32\n",
    "learning_rate = 0.001\n",
    "num_epochs = 100\n",
    "log_every_n_epochs = 10"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's load some dataset from scikit-learn."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X, y = fetch_california_housing(return_X_y=True)\n",
    "X = X.astype(\"float32\")\n",
    "y = y.astype(\"float32\")\n",
    "num_features = X.shape[1]\n",
    "X.shape, y.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's split our data. For neural networks, we need 3 datasets: training, validation and testing datasets."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X_train_valid, X_test, y_train_valid, y_test = train_test_split(\n",
    "    X, y, train_size=0.8, random_state=42\n",
    ")\n",
    "\n",
    "scaler = StandardScaler()\n",
    "X_train_valid = scaler.fit_transform(X_train_valid)\n",
    "X_test = scaler.transform(X_test)\n",
    "\n",
    "X_train, X_valid, y_train, y_valid = train_test_split(\n",
    "    X_train_valid, y_train_valid, train_size=0.75, random_state=42\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "print(\"Training set shapes (X, y):\", X_train.shape, y_train.shape)\n",
    "print(\"Validation set shapes (X, y):\", X_valid.shape, y_valid.shape)\n",
    "print(\"Testing set shapes (X, y):\", X_test.shape, y_test.shape)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's train a simple SVM for regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf = SVR()\n",
    "clf.fit(X_train_valid, y_train_valid)\n",
    "prediction = clf.predict(X_test)\n",
    "print(\"MSE:\", mean_squared_error(y_test, prediction))\n",
    "print(\"R^2:\", r2_score(y_test, prediction))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now, we will train neural networks using the PyTorch library."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_dataset = TensorDataset(torch.from_numpy(X_train), torch.from_numpy(y_train))\n",
    "valid_dataset = TensorDataset(torch.from_numpy(X_valid), torch.from_numpy(y_valid))\n",
    "test_dataset = TensorDataset(torch.from_numpy(X_test), torch.from_numpy(y_test))\n",
    "\n",
    "train_loader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True)\n",
    "valid_loader = DataLoader(valid_dataset, batch_size=batch_size)\n",
    "test_loader = DataLoader(test_dataset, batch_size=batch_size)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def pytorch_train_one_epoch(pytorch_network, optimizer, loss_function):\n",
    "    \"\"\"\n",
    "    Trains the neural network for one epoch on the train DataLoader.\n",
    "\n",
    "    Args:\n",
    "        pytorch_network (torch.nn.Module): The neural network to train.\n",
    "        optimizer (torch.optim.Optimizer): The optimizer of the neural network\n",
    "        loss_function: The loss function.\n",
    "\n",
    "    Returns:\n",
    "        A tuple (loss, accuracy) corresponding to an average of the losses and\n",
    "        an average of the accuracy, respectively, on the train DataLoader.\n",
    "    \"\"\"\n",
    "    pytorch_network.train(True)\n",
    "    with torch.enable_grad():\n",
    "        loss_sum = 0.0\n",
    "        example_count = 0\n",
    "        for (x, y) in train_loader:\n",
    "            # Transfer batch on GPU if needed.\n",
    "            x = x.to(device)\n",
    "            y = y.to(device)\n",
    "\n",
    "            # We need to zero the gradient before every batch because the new\n",
    "            # gradients would otherwise be summed with the previous gradients.\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "            # Compute the predictions of the neural network on the batch.\n",
    "            y_pred = pytorch_network(x)\n",
    "\n",
    "            loss = loss_function(y_pred, y)\n",
    "\n",
    "            # Do the the backpropagation to compute the gradients of the parameters.\n",
    "            loss.backward()\n",
    "\n",
    "            # Update our parameters with the gradient.\n",
    "            optimizer.step()\n",
    "\n",
    "            # Since the loss and accuracy are averages for the batch, we multiply\n",
    "            # them by the the number of examples so that we can do the right\n",
    "            # averages at the end of the epoch.\n",
    "            loss_sum += float(loss) * len(x)\n",
    "            example_count += len(x)\n",
    "\n",
    "    avg_loss = loss_sum / example_count\n",
    "    return avg_loss\n",
    "\n",
    "\n",
    "def pytorch_test(pytorch_network, loader, loss_function):\n",
    "    \"\"\"\n",
    "    Tests the neural network on a DataLoader.\n",
    "\n",
    "    Args:\n",
    "        pytorch_network (torch.nn.Module): The neural network to test.\n",
    "        loader (torch.utils.data.DataLoader): The DataLoader to test on.\n",
    "        loss_function: The loss function.\n",
    "\n",
    "    Returns:\n",
    "        A tuple (loss, accuracy) corresponding to an average of the losses and\n",
    "        an average of the accuracy, respectively, on the DataLoader.\n",
    "    \"\"\"\n",
    "    pytorch_network.eval()\n",
    "    with torch.no_grad():\n",
    "        loss_sum = 0.0\n",
    "        example_count = 0\n",
    "        for (x, y) in loader:\n",
    "            # Transfer batch on GPU if needed.\n",
    "            x = x.to(device)\n",
    "            y = y.to(device)\n",
    "\n",
    "            y_pred = pytorch_network(x)\n",
    "            loss = loss_function(y_pred, y)\n",
    "\n",
    "            # Since the loss and accuracy are averages for the batch, we multiply\n",
    "            # them by the the number of examples so that we can do the right\n",
    "            # averages at the end of the test.\n",
    "            loss_sum += float(loss) * len(x)\n",
    "            example_count += len(x)\n",
    "\n",
    "    avg_loss = loss_sum / example_count\n",
    "    return avg_loss\n",
    "\n",
    "\n",
    "def pytorch_train(pytorch_network):\n",
    "    \"\"\"\n",
    "    This function transfers the neural network to the right device,\n",
    "    trains it for a certain number of epochs, tests at each epoch on\n",
    "    the validation set and outputs the results on the test set at the\n",
    "    end of training.\n",
    "\n",
    "    Args:\n",
    "        pytorch_network (torch.nn.Module): The neural network to train.\n",
    "    \"\"\"\n",
    "    print(\"Network:\")\n",
    "    print(pytorch_network)\n",
    "    print()\n",
    "\n",
    "    # Transfer weights on GPU if needed.\n",
    "    pytorch_network.to(device)\n",
    "\n",
    "    loss_function = nn.MSELoss()\n",
    "    optimizer = optim.Adam(pytorch_network.parameters(), lr=learning_rate)\n",
    "\n",
    "    for epoch in range(1, num_epochs + 1):\n",
    "        # Training the neural network via backpropagation\n",
    "        train_loss = pytorch_train_one_epoch(pytorch_network, optimizer, loss_function)\n",
    "\n",
    "        # Validation at the end of the epoch\n",
    "        valid_loss = pytorch_test(pytorch_network, valid_loader, loss_function)\n",
    "\n",
    "        if epoch % log_every_n_epochs == 0:\n",
    "            print(\n",
    "                \"Epoch {}/{}: loss: {}, val_loss: {}\".format(\n",
    "                    epoch, num_epochs, train_loss, valid_loss\n",
    "                )\n",
    "            )\n",
    "\n",
    "    # Test at the end of the training\n",
    "    test_loss = pytorch_test(pytorch_network, test_loader, loss_function)\n",
    "    print(\"Test Loss: {}\".format(test_loss))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A simple regression."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "fc_net = nn.Sequential(\n",
    "    nn.Linear(num_features, 1),\n",
    "    nn.Flatten(0),\n",
    ")\n",
    "pytorch_train(fc_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A neural network with 2 layers and no activation function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fc_net = nn.Sequential(\n",
    "    nn.Linear(num_features, 100),\n",
    "    nn.Linear(100, 1),\n",
    "    nn.Flatten(0),\n",
    ")\n",
    "pytorch_train(fc_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's add an activation function."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fc_net = nn.Sequential(\n",
    "    nn.Linear(num_features, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 1),\n",
    "    nn.Flatten(0),\n",
    ")\n",
    "pytorch_train(fc_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's add a third layer."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fc_net = nn.Sequential(\n",
    "    nn.Linear(num_features, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 1),\n",
    "    nn.Flatten(0),\n",
    ")\n",
    "pytorch_train(fc_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "And a fourth."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "fc_net = nn.Sequential(\n",
    "    nn.Linear(num_features, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 1),\n",
    "    nn.Flatten(0),\n",
    ")\n",
    "pytorch_train(fc_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's use the Poutyne library instead of our own functions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def poutyne_train(pytorch_network):\n",
    "    \"\"\"\n",
    "    This function creates a Poutyne Model (see https://poutyne.org/model.html), sends the Model\n",
    "    on the specified device, and uses the `fit_generator` method to train the neural network.\n",
    "    At the end, the `evaluate_generator` is used on  the test set.\n",
    "\n",
    "    Args:\n",
    "        pytorch_network (torch.nn.Module): The neural network to train.\n",
    "    \"\"\"\n",
    "    print(pytorch_network)\n",
    "\n",
    "    optimizer = optim.Adam(pytorch_network.parameters(), lr=learning_rate)\n",
    "    loss_function = nn.MSELoss()\n",
    "\n",
    "    # Poutyne Model on GPU\n",
    "    model = Model(\n",
    "        pytorch_network,\n",
    "        optimizer,\n",
    "        loss_function,\n",
    "        batch_metrics=[\"l1\", torchmetrics.R2Score()],\n",
    "        device=device,\n",
    "    )\n",
    "\n",
    "    # Train\n",
    "    history = model.fit(\n",
    "        X_train,\n",
    "        y_train,\n",
    "        validation_data=(X_valid, y_valid),\n",
    "        epochs=num_epochs,\n",
    "        verbose=False,\n",
    "        callbacks=[\n",
    "            EpochProgressionCallback(show_every_n_epochs=log_every_n_epochs),\n",
    "            # BestModelRestore(verbose=True)\n",
    "        ],\n",
    "    )\n",
    "\n",
    "    # Test\n",
    "    test_loss = model.evaluate(\n",
    "        X_test, y_test, progress_options=dict(show_every_n_test_steps=\"none\")\n",
    "    )\n",
    "\n",
    "    return history, test_loss"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "fc_net = nn.Sequential(\n",
    "    nn.Linear(num_features, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 100),\n",
    "    nn.ReLU(),\n",
    "    nn.Linear(100, 1),\n",
    "    nn.Flatten(0),\n",
    ")\n",
    "history, test_loss = poutyne_train(fc_net)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Let's plot our losses and metrics."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "from poutyne import plot_history\n",
    "\n",
    "plot_history(\n",
    "    history, metrics=[\"loss\", \"l1\", \"r2_score\"], labels=[\"Loss\", \"L1\", \"$R^2$\"]\n",
    ")\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}